__init__.py 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import logging
  18. import logging.config
  19. import os
  20. import os.path
  21. import re
  22. import sys
  23. from io import StringIO
  24. from logging import FileHandler
  25. from logging.handlers import RotatingFileHandler
  26. from desktop.lib.paths import get_desktop_root
  27. from desktop.log import formatter
  28. from desktop.log.formatter import MessageOnlyFormatter
  29. DEFAULT_LOG_DIR = 'logs'
  30. LOG_FORMAT = '[%(asctime)s] %(module)-12s %(levelname)-8s %(message)s'
  31. DATE_FORMAT = '%d/%b/%Y %H:%M:%S %z'
  32. FORCE_DEBUG = False
  33. CONF_RE = re.compile('%LOG_DIR%|%PROC_NAME%')
  34. _log_dir = None
  35. lib_dir = os.path.dirname(os.path.realpath(__file__)) + "/../"
  36. def _read_log_conf(proc_name, log_dir):
  37. """
  38. _read_log_conf(proc_name, log_dir) -> StringIO or None
  39. This method also replaces the %LOG_DIR% and %PROC_NAME% occurrences.
  40. """
  41. def _repl(match):
  42. if match.group(0) == '%LOG_DIR%':
  43. return log_dir
  44. elif match.group(0) == '%PROC_NAME%':
  45. return proc_name
  46. log_conf = lib_dir + 'log.conf'
  47. if not os.path.isfile(log_conf):
  48. return None
  49. try:
  50. raw = file(log_conf).read()
  51. sio = StringIO(CONF_RE.sub(_repl, raw))
  52. return sio
  53. except IOError as ex:
  54. print >> sys.stderr, "ERROR: Failed to open %s: %s" % (log_conf, ex)
  55. return None
  56. def _find_console_stream_handler(logger):
  57. """Find a StreamHandler that is attached to the logger that prints to the console."""
  58. for handler in logger.handlers:
  59. if isinstance(handler, logging.StreamHandler) and handler.stream in (sys.stderr, sys.stdout):
  60. return logger
  61. return None
  62. class AuditHandler(RotatingFileHandler):
  63. pass
  64. def get_audit_logger():
  65. from desktop.conf import AUDIT_EVENT_LOG_DIR, AUDIT_LOG_MAX_FILE_SIZE
  66. audit_logger = logging.getLogger('audit')
  67. if not filter(lambda hclass: isinstance(hclass, AuditHandler), audit_logger.handlers): # Don't add handler twice
  68. size, unit = int(AUDIT_LOG_MAX_FILE_SIZE.get()[:-2]), AUDIT_LOG_MAX_FILE_SIZE.get()[-2:]
  69. maxBytes = size * 1024 ** (1 if unit == 'KB' else 2 if unit == 'MB' else 3)
  70. audit_handler = AuditHandler(AUDIT_EVENT_LOG_DIR.get(), maxBytes=maxBytes, backupCount=50)
  71. audit_handler.setFormatter(MessageOnlyFormatter())
  72. audit_logger.addHandler(audit_handler)
  73. return audit_logger
  74. def chown_log_dir(uid, gid):
  75. """
  76. chown all files in the log dir to this user and group.
  77. Should only be called after loggic has been setup.
  78. Return success
  79. """
  80. if _log_dir is None:
  81. return False
  82. try:
  83. os.chown(_log_dir, uid, gid)
  84. for entry in os.listdir(_log_dir):
  85. os.chown(os.path.join(_log_dir, entry), uid, gid)
  86. return True
  87. except OSError as ex:
  88. print >> sys.stderr, 'Failed to chown log directory %s: ex' % (_log_dir, ex)
  89. return False
  90. def basic_logging(proc_name, log_dir=None):
  91. """
  92. Configure logging for the program ``proc_name``:
  93. - Apply log.conf in the config directory.
  94. - If DESKTOP_LOGLEVEL environment variable is specified, the root console
  95. handler (stdout/stderr) is set to that level. If there is no console handler,
  96. a new one is created.
  97. - Defining the environment variable DESKTOP_DEBUG is the same as setting
  98. DESKTOP_LOGLEVEL=DEBUG.
  99. The ``log_dir`` will replace the %LOG_DIR% in log.conf. If not specified, we look
  100. for the DESTKOP_LOG_DIR environment variable, and then default to the DEFAULT_LOG_DIR.
  101. This removes all previously installed logging handlers.
  102. """
  103. global FORCE_DEBUG
  104. # Setup log_dir
  105. if not log_dir:
  106. log_dir = os.getenv("DESKTOP_LOG_DIR", DEFAULT_LOG_DIR)
  107. if not os.path.exists(log_dir):
  108. try:
  109. os.makedirs(log_dir)
  110. except OSError as err:
  111. print >> sys.stderr, 'Failed to create log directory "%s": %s' % (log_dir, err)
  112. raise err
  113. # Remember where our log directory is
  114. global _log_dir
  115. _log_dir = log_dir
  116. log_conf = _read_log_conf(proc_name, log_dir)
  117. if log_conf is not None:
  118. logging.config.fileConfig(log_conf)
  119. root_logger = logging.getLogger()
  120. else:
  121. # Get rid of any preinstalled/default handlers
  122. root_logger = logging.getLogger()
  123. for h in root_logger.handlers:
  124. root_logger.removeHandler(h)
  125. # always keep DEBUG at the root, since we'll filter in the
  126. # handlers themselves - this allows the /logs endpoint
  127. # to always have all logs.
  128. root_logger.setLevel(logging.DEBUG)
  129. # Handle env variables
  130. env_loglevel = os.getenv("DESKTOP_LOGLEVEL")
  131. env_debug = os.getenv('DESKTOP_DEBUG') or FORCE_DEBUG
  132. if env_debug:
  133. env_loglevel = 'DEBUG'
  134. if env_loglevel:
  135. try:
  136. lvl = getattr(logging, env_loglevel.upper())
  137. except AttributeError:
  138. raise Exception("Invalid log level in DESKTOP_LOGLEVEL: %s" % (env_loglevel,))
  139. # Set the StreamHandler to the level (create one if necessary)
  140. handler = _find_console_stream_handler(root_logger)
  141. if not handler:
  142. handler = logging.StreamHandler()
  143. handler.setFormatter(logging.Formatter(LOG_FORMAT, DATE_FORMAT))
  144. root_logger.addHandler(handler)
  145. handler.setLevel(lvl)
  146. # Set all loggers but error.log to the same logging level
  147. error_handler = logging.getLogger('handler_logfile')
  148. for h in root_logger.handlers:
  149. if isinstance(h, (FileHandler, RotatingFileHandler)) and h != error_handler:
  150. h.setLevel(lvl)
  151. def fancy_logging():
  152. """Configure logging into a buffer for /logs endpoint."""
  153. from log_buffer import FixedBufferHandler
  154. BUFFER_SIZE = 500 * 200 # This is the size in characters, not bytes. Targets about 500 rows.
  155. buffer_handler = FixedBufferHandler(BUFFER_SIZE)
  156. _formatter = formatter.Formatter(LOG_FORMAT, DATE_FORMAT)
  157. # We always want to catch all messages in our error report buffer
  158. buffer_handler.setLevel(logging.DEBUG)
  159. buffer_handler.setFormatter(_formatter)
  160. root_logger = logging.getLogger()
  161. root_logger.addHandler(buffer_handler)
  162. def get_all_debug():
  163. global FORCE_DEBUG
  164. return FORCE_DEBUG
  165. def set_all_debug():
  166. from desktop.settings import ENV_HUE_PROCESS_NAME # Circular dependency
  167. global FORCE_DEBUG
  168. FORCE_DEBUG = True
  169. basic_logging(os.environ[ENV_HUE_PROCESS_NAME])
  170. fancy_logging()
  171. def reset_all_debug():
  172. from desktop.settings import ENV_HUE_PROCESS_NAME # Circular dependency
  173. global FORCE_DEBUG
  174. FORCE_DEBUG = False
  175. basic_logging(os.environ[ENV_HUE_PROCESS_NAME])
  176. fancy_logging()