# Licensed to Cloudera, Inc. under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. Cloudera, Inc. licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. diff --git a/apps/filebrowser/src/filebrowser/settings.py b/apps/filebrowser/src/filebrowser/settings.py index cb7a8c7c3d..9a783965a8 100644 --- a/apps/filebrowser/src/filebrowser/settings.py +++ b/apps/filebrowser/src/filebrowser/settings.py @@ -22,7 +22,7 @@ ICON = "filebrowser/art/icon_filebrowser_48.png" MENU_INDEX = 20 from aws.s3.s3fs import PERMISSION_ACTION_S3 -from azure.adls.webhdfs import PERMISSION_ACTION_ADLS +PERMISSION_ACTION_ADLS = "adls_access" PERMISSION_ACTIONS = ( diff --git a/desktop/core/src/desktop/conf.py b/desktop/core/src/desktop/conf.py index 6a84e9ca28..6095fad1c6 100644 --- a/desktop/core/src/desktop/conf.py +++ b/desktop/core/src/desktop/conf.py @@ -397,6 +397,18 @@ COLLECT_USAGE = Config( type=coerce_bool, default=True) +REST_RESPONSE_SIZE = Config( + key="rest_response_size", + help=_("Number of characters the rest api reponse calls to dump to the logs when debug is enabled."), + type=int, + default=1000) + +THRIFT_RESPONSE_SIZE = Config( + key="thrift_response_size", + help=_("Number of characters the thrift api reponse calls to dump to the logs when debug is enabled."), + type=int, + default=1000) + LEAFLET_TILE_LAYER = Config( key="leaflet_tile_layer", help=_("Tile layer server URL for the Leaflet map charts. Read more on http://leafletjs.com/reference.html#tilelayer. Make sure you add the tile domain to the img-src section of the 'secure_content_security_policy' configuration parameter as well."), @@ -1338,6 +1350,17 @@ MEMORY_PROFILER = Config( default=False) +def get_instrumentation_default(): + """If django_debug_mode is True, this is automatically enabled""" + return DJANGO_DEBUG_MODE.get() + +INSTRUMENTATION = Config( + key='instrumentation', + help=_('Enable or disable instrumentation. If django_debug_mode is True, this is automatically enabled.'), + type=coerce_bool, + dynamic_default=get_instrumentation_default) + + AUDIT_EVENT_LOG_DIR = Config( key="audit_event_log_dir", help=_("The directory where to store the auditing logs. Auditing is disable if the value is empty."), diff --git a/desktop/core/src/desktop/lib/rest/resource.py b/desktop/core/src/desktop/lib/rest/resource.py index 04644a502b..0310611b38 100644 --- a/desktop/core/src/desktop/lib/rest/resource.py +++ b/desktop/core/src/desktop/lib/rest/resource.py @@ -18,9 +18,13 @@ import logging import posixpath import time +from django.utils.encoding import iri_to_uri, smart_str +from django.utils.http import urlencode + from desktop.lib.i18n import smart_unicode from desktop.lib.apputil import WARN_LEVEL_CALL_DURATION_MS, INFO_LEVEL_CALL_DURATION_MS +from desktop import conf LOG = logging.getLogger(__name__) @@ -98,13 +102,20 @@ class Resource(object): if log_response: duration = time.time() - start_time - message = "%s %s Got response%s: %s%s" % ( - method, - smart_unicode(path, errors='ignore'), - ' in %dms' % (duration * 1000), - smart_unicode(resp.content[:1000], errors='replace'), - len(resp.content) > 1000 and "..." or "" - ) + log_length = conf.REST_RESPONSE_SIZE.get() != -1 and conf.REST_RESPONSE_SIZE.get() # We want to output duration without content + message = '%s %s %s%s%s %s%s returned in %dms %s %s %s%s' % ( + method, + type(self._client._session.auth) if self._client._session and self._client._session.auth else None, + self._client._base_url, + smart_str(path), + iri_to_uri('?' + urlencode(params)) if params else '', + smart_unicode(data, errors='replace')[:log_length] if data else "", + log_length and len(data) > log_length and "..." or "" if data else "", + (duration * 1000), + resp.status_code if resp else 0, + len(resp.content) if resp else 0, + smart_unicode(resp.content[:log_length], errors='replace') if resp else "", + log_length and len(resp.content) > log_length and "..." or "" if resp else "") log_if_slow_call(duration=duration, message=message, logger=self._client.logger) return resp @@ -199,4 +210,4 @@ def log_if_slow_call(duration, message, logger): elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000: logger.info('SLOW: %.2f - %s' % (duration, message)) else: - logger.debug(message) + logging.debug(message) diff --git a/desktop/core/src/desktop/lib/thrift_util.py b/desktop/core/src/desktop/lib/thrift_util.py index d6ab2a3e74..2f6503161c 100644 --- a/desktop/core/src/desktop/lib/thrift_util.py +++ b/desktop/core/src/desktop/lib/thrift_util.py @@ -44,6 +44,7 @@ from desktop.lib.thrift_.TSSLSocketWithWildcardSAN import TSSLSocketWithWildcard from desktop.lib.thrift_sasl import TSaslClientTransport from desktop.lib.exceptions import StructuredException, StructuredThriftTransportException +from desktop import conf LOG = logging.getLogger(__name__) @@ -441,7 +442,7 @@ class SuperClient(object): log_msg = _unpack_guid_secret_in_handle(repr(ret)) # Truncate log message, increase output in DEBUG mode - log_limit = 2000 if settings.DEBUG else 1000 + log_limit = conf.THRIFT_RESPONSE_SIZE.get() if settings.DEBUG else 1000 log_msg = log_msg[:log_limit] + (log_msg[log_limit:] and '...') duration = time.time() - st @@ -755,4 +756,4 @@ def log_if_slow_call(duration, message): elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000: LOG.info('SLOW: %.2f - %s' % (duration, message)) else: - LOG.debug(message) + logging.debug(message) diff --git a/desktop/core/src/desktop/log/access.py b/desktop/core/src/desktop/log/access.py index 679f9d1035..ad3c48805f 100644 --- a/desktop/core/src/desktop/log/access.py +++ b/desktop/core/src/desktop/log/access.py @@ -22,6 +22,8 @@ This assumes a single-threaded server. import logging import re +import resource +import sys import threading import time @@ -81,15 +83,36 @@ class AccessInfo(dict): self['agent'] = request.META.get('HTTP_USER_AGENT', '-') self['time'] = time.time() self['duration'] = None + self['memory'] = None + + def memory_usage_resource(self): + """ + This is a lightweight way to get the total peak memory as + doing the diffing before/after request with guppy was too inconsistent and memory intensive. + """ + rusage_denom = 1024 + if sys.platform == 'darwin': + rusage_denom = rusage_denom * 1024 + # get peak memory usage, bytes on OSX, Kilobytes on Linux + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom def log(self, level, msg=None, start_time=None): - self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None else '' + is_instrumentation = desktop.conf.INSTRUMENTATION.get() + self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None and is_instrumentation else '' + self['memory'] = ' (mem: %dmb)' % self.memory_usage_resource() if is_instrumentation else '' if msg is not None: self['msg'] = msg - ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s -- %(msg)s' % self) + ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s-- %(msg)s' % self) else: - ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s' % self) + ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s' % self) + + if is_instrumentation: + import gc + gc.collect() + for o in gc.garbage: + for r in gc.get_referrers(o): + ACCESS_LOG.log(level, 'ref for %r: %r' % (o, r)) def add_to_access_history(self, app): """Record this user access to the recent access map""" diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py index db89c9dfb0..62c26e2b07 100644 --- a/desktop/core/src/desktop/settings.py +++ b/desktop/core/src/desktop/settings.py @@ -20,6 +20,7 @@ # Local customizations are done by symlinking a file # as local_settings.py. +import gc import logging import os import pkg_resources @@ -509,6 +510,9 @@ if desktop.conf.MEMORY_PROFILER.get(): MEMORY_PROFILER = hpy() MEMORY_PROFILER.setrelheap() +# Instrumentation +if desktop.conf.INSTRUMENTATION.get(): + gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_OBJECTS) if not desktop.conf.DATABASE_LOGGING.get(): def disable_database_logging():