5.15_all_logging.patch 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. diff --git a/apps/filebrowser/src/filebrowser/settings.py b/apps/filebrowser/src/filebrowser/settings.py
  17. index cb7a8c7c3d..9a783965a8 100644
  18. --- a/apps/filebrowser/src/filebrowser/settings.py
  19. +++ b/apps/filebrowser/src/filebrowser/settings.py
  20. @@ -22,7 +22,7 @@ ICON = "filebrowser/art/icon_filebrowser_48.png"
  21. MENU_INDEX = 20
  22. from aws.s3.s3fs import PERMISSION_ACTION_S3
  23. -from azure.adls.webhdfs import PERMISSION_ACTION_ADLS
  24. +PERMISSION_ACTION_ADLS = "adls_access"
  25. PERMISSION_ACTIONS = (
  26. diff --git a/desktop/core/src/desktop/conf.py b/desktop/core/src/desktop/conf.py
  27. index 6a84e9ca28..6095fad1c6 100644
  28. --- a/desktop/core/src/desktop/conf.py
  29. +++ b/desktop/core/src/desktop/conf.py
  30. @@ -397,6 +397,18 @@ COLLECT_USAGE = Config(
  31. type=coerce_bool,
  32. default=True)
  33. +REST_RESPONSE_SIZE = Config(
  34. + key="rest_response_size",
  35. + help=_("Number of characters the rest api reponse calls to dump to the logs when debug is enabled."),
  36. + type=int,
  37. + default=1000)
  38. +
  39. +THRIFT_RESPONSE_SIZE = Config(
  40. + key="thrift_response_size",
  41. + help=_("Number of characters the thrift api reponse calls to dump to the logs when debug is enabled."),
  42. + type=int,
  43. + default=1000)
  44. +
  45. LEAFLET_TILE_LAYER = Config(
  46. key="leaflet_tile_layer",
  47. help=_("Tile layer server URL for the Leaflet map charts. Read more on http://leafletjs.com/reference.html#tilelayer. Make sure you add the tile domain to the img-src section of the 'secure_content_security_policy' configuration parameter as well."),
  48. @@ -1338,6 +1350,17 @@ MEMORY_PROFILER = Config(
  49. default=False)
  50. +def get_instrumentation_default():
  51. + """If django_debug_mode is True, this is automatically enabled"""
  52. + return DJANGO_DEBUG_MODE.get()
  53. +
  54. +INSTRUMENTATION = Config(
  55. + key='instrumentation',
  56. + help=_('Enable or disable instrumentation. If django_debug_mode is True, this is automatically enabled.'),
  57. + type=coerce_bool,
  58. + dynamic_default=get_instrumentation_default)
  59. +
  60. +
  61. AUDIT_EVENT_LOG_DIR = Config(
  62. key="audit_event_log_dir",
  63. help=_("The directory where to store the auditing logs. Auditing is disable if the value is empty."),
  64. diff --git a/desktop/core/src/desktop/lib/rest/resource.py b/desktop/core/src/desktop/lib/rest/resource.py
  65. index 04644a502b..0310611b38 100644
  66. --- a/desktop/core/src/desktop/lib/rest/resource.py
  67. +++ b/desktop/core/src/desktop/lib/rest/resource.py
  68. @@ -18,9 +18,13 @@ import logging
  69. import posixpath
  70. import time
  71. +from django.utils.encoding import iri_to_uri, smart_str
  72. +from django.utils.http import urlencode
  73. +
  74. from desktop.lib.i18n import smart_unicode
  75. from desktop.lib.apputil import WARN_LEVEL_CALL_DURATION_MS, INFO_LEVEL_CALL_DURATION_MS
  76. +from desktop import conf
  77. LOG = logging.getLogger(__name__)
  78. @@ -98,13 +102,20 @@ class Resource(object):
  79. if log_response:
  80. duration = time.time() - start_time
  81. - message = "%s %s Got response%s: %s%s" % (
  82. - method,
  83. - smart_unicode(path, errors='ignore'),
  84. - ' in %dms' % (duration * 1000),
  85. - smart_unicode(resp.content[:1000], errors='replace'),
  86. - len(resp.content) > 1000 and "..." or ""
  87. - )
  88. + log_length = conf.REST_RESPONSE_SIZE.get() != -1 and conf.REST_RESPONSE_SIZE.get() # We want to output duration without content
  89. + message = '%s %s %s%s%s %s%s returned in %dms %s %s %s%s' % (
  90. + method,
  91. + type(self._client._session.auth) if self._client._session and self._client._session.auth else None,
  92. + self._client._base_url,
  93. + smart_str(path),
  94. + iri_to_uri('?' + urlencode(params)) if params else '',
  95. + smart_unicode(data, errors='replace')[:log_length] if data else "",
  96. + log_length and len(data) > log_length and "..." or "" if data else "",
  97. + (duration * 1000),
  98. + resp.status_code if resp else 0,
  99. + len(resp.content) if resp else 0,
  100. + smart_unicode(resp.content[:log_length], errors='replace') if resp else "",
  101. + log_length and len(resp.content) > log_length and "..." or "" if resp else "")
  102. log_if_slow_call(duration=duration, message=message, logger=self._client.logger)
  103. return resp
  104. @@ -199,4 +210,4 @@ def log_if_slow_call(duration, message, logger):
  105. elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000:
  106. logger.info('SLOW: %.2f - %s' % (duration, message))
  107. else:
  108. - logger.debug(message)
  109. + logging.debug(message)
  110. diff --git a/desktop/core/src/desktop/lib/thrift_util.py b/desktop/core/src/desktop/lib/thrift_util.py
  111. index d6ab2a3e74..2f6503161c 100644
  112. --- a/desktop/core/src/desktop/lib/thrift_util.py
  113. +++ b/desktop/core/src/desktop/lib/thrift_util.py
  114. @@ -44,6 +44,7 @@ from desktop.lib.thrift_.TSSLSocketWithWildcardSAN import TSSLSocketWithWildcard
  115. from desktop.lib.thrift_sasl import TSaslClientTransport
  116. from desktop.lib.exceptions import StructuredException, StructuredThriftTransportException
  117. +from desktop import conf
  118. LOG = logging.getLogger(__name__)
  119. @@ -441,7 +442,7 @@ class SuperClient(object):
  120. log_msg = _unpack_guid_secret_in_handle(repr(ret))
  121. # Truncate log message, increase output in DEBUG mode
  122. - log_limit = 2000 if settings.DEBUG else 1000
  123. + log_limit = conf.THRIFT_RESPONSE_SIZE.get() if settings.DEBUG else 1000
  124. log_msg = log_msg[:log_limit] + (log_msg[log_limit:] and '...')
  125. duration = time.time() - st
  126. @@ -755,4 +756,4 @@ def log_if_slow_call(duration, message):
  127. elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000:
  128. LOG.info('SLOW: %.2f - %s' % (duration, message))
  129. else:
  130. - LOG.debug(message)
  131. + logging.debug(message)
  132. diff --git a/desktop/core/src/desktop/log/access.py b/desktop/core/src/desktop/log/access.py
  133. index 679f9d1035..ad3c48805f 100644
  134. --- a/desktop/core/src/desktop/log/access.py
  135. +++ b/desktop/core/src/desktop/log/access.py
  136. @@ -22,6 +22,8 @@ This assumes a single-threaded server.
  137. import logging
  138. import re
  139. +import resource
  140. +import sys
  141. import threading
  142. import time
  143. @@ -81,15 +83,36 @@ class AccessInfo(dict):
  144. self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
  145. self['time'] = time.time()
  146. self['duration'] = None
  147. + self['memory'] = None
  148. +
  149. + def memory_usage_resource(self):
  150. + """
  151. + This is a lightweight way to get the total peak memory as
  152. + doing the diffing before/after request with guppy was too inconsistent and memory intensive.
  153. + """
  154. + rusage_denom = 1024
  155. + if sys.platform == 'darwin':
  156. + rusage_denom = rusage_denom * 1024
  157. + # get peak memory usage, bytes on OSX, Kilobytes on Linux
  158. + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
  159. def log(self, level, msg=None, start_time=None):
  160. - self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None else ''
  161. + is_instrumentation = desktop.conf.INSTRUMENTATION.get()
  162. + self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None and is_instrumentation else ''
  163. + self['memory'] = ' (mem: %dmb)' % self.memory_usage_resource() if is_instrumentation else ''
  164. if msg is not None:
  165. self['msg'] = msg
  166. - ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s -- %(msg)s' % self)
  167. + ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s-- %(msg)s' % self)
  168. else:
  169. - ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s' % self)
  170. + ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s' % self)
  171. +
  172. + if is_instrumentation:
  173. + import gc
  174. + gc.collect()
  175. + for o in gc.garbage:
  176. + for r in gc.get_referrers(o):
  177. + ACCESS_LOG.log(level, 'ref for %r: %r' % (o, r))
  178. def add_to_access_history(self, app):
  179. """Record this user access to the recent access map"""
  180. diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py
  181. index db89c9dfb0..62c26e2b07 100644
  182. --- a/desktop/core/src/desktop/settings.py
  183. +++ b/desktop/core/src/desktop/settings.py
  184. @@ -20,6 +20,7 @@
  185. # Local customizations are done by symlinking a file
  186. # as local_settings.py.
  187. +import gc
  188. import logging
  189. import os
  190. import pkg_resources
  191. @@ -509,6 +510,9 @@ if desktop.conf.MEMORY_PROFILER.get():
  192. MEMORY_PROFILER = hpy()
  193. MEMORY_PROFILER.setrelheap()
  194. +# Instrumentation
  195. +if desktop.conf.INSTRUMENTATION.get():
  196. + gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_OBJECTS)
  197. if not desktop.conf.DATABASE_LOGGING.get():
  198. def disable_database_logging():