settings.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. # Django settings for Hue.
  18. #
  19. # Local customizations are done by symlinking a file
  20. # as local_settings.py.
  21. from builtins import map, zip
  22. import datetime
  23. import gc
  24. import json
  25. import logging
  26. import os
  27. import pkg_resources
  28. import sys
  29. import uuid
  30. import django_opentracing
  31. from django.utils.translation import ugettext_lazy as _
  32. import desktop.redaction
  33. from desktop.lib.paths import get_desktop_root, get_run_root
  34. from desktop.lib.python_util import force_dict_to_strings
  35. from aws.conf import is_enabled as is_s3_enabled
  36. from azure.conf import is_abfs_enabled
  37. # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
  38. BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), '..', '..', '..'))
  39. HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown"
  40. NICE_NAME = "Hue"
  41. ENV_HUE_PROCESS_NAME = "HUE_PROCESS_NAME"
  42. ENV_DESKTOP_DEBUG = "DESKTOP_DEBUG"
  43. LOGGING_CONFIG = None # We're handling our own logging config. Consider upgrading our logging infra to LOGGING_CONFIG
  44. ############################################################
  45. # Part 1: Logging and imports.
  46. ############################################################
  47. # Configure debug mode
  48. DEBUG = True
  49. GTEMPLATE_DEBUG = DEBUG
  50. # Start basic logging as soon as possible.
  51. if ENV_HUE_PROCESS_NAME not in os.environ:
  52. _proc = os.path.basename(len(sys.argv) > 1 and sys.argv[1] or sys.argv[0])
  53. os.environ[ENV_HUE_PROCESS_NAME] = _proc
  54. desktop.log.basic_logging(os.environ[ENV_HUE_PROCESS_NAME])
  55. logging.info("Welcome to Hue " + HUE_DESKTOP_VERSION)
  56. # Then we can safely import some more stuff
  57. from desktop import appmanager
  58. from desktop.lib import conf
  59. # Add fancy logging
  60. desktop.log.fancy_logging()
  61. ############################################################
  62. # Part 2: Generic Configuration
  63. ############################################################
  64. # Language code for this installation. All choices can be found here:
  65. # http://www.i18nguy.com/unicode/language-identifiers.html
  66. LANGUAGE_CODE = 'en-us'
  67. LANGUAGES = [
  68. ('de', _('German')),
  69. ('en-us', _('English')),
  70. ('es', _('Spanish')),
  71. ('fr', _('French')),
  72. ('ja', _('Japanese')),
  73. ('ko', _('Korean')),
  74. ('pt', _('Portuguese')),
  75. ('pt_BR', _('Brazilian Portuguese')),
  76. ('zh-CN', _('Simplified Chinese')),
  77. ]
  78. SITE_ID = 1
  79. # If you set this to False, Django will make some optimizations so as not
  80. # to load the internationalization machinery.
  81. USE_I18N = True
  82. # If you set this to False, Django will not format dates, numbers and
  83. # calendars according to the current locale.
  84. USE_L10N = True
  85. # If you set this to False, Django will not use timezone-aware datetimes.
  86. USE_TZ = False
  87. # URL that handles the media served from MEDIA_ROOT. Make sure to use a
  88. # trailing slash.
  89. # Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
  90. MEDIA_URL = ''
  91. ############################################################
  92. # Part 3: Django configuration
  93. ############################################################
  94. # Additional locations of static files
  95. STATICFILES_DIRS = (
  96. os.path.join(BASE_DIR, 'desktop', 'libs', 'indexer', 'src', 'indexer', 'static'),
  97. os.path.join(BASE_DIR, 'desktop', 'libs', 'notebook', 'src', 'notebook', 'static'),
  98. os.path.join(BASE_DIR, 'desktop', 'libs', 'liboauth', 'src', 'liboauth', 'static'),
  99. )
  100. STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.CachedStaticFilesStorage'
  101. # For Django admin interface
  102. STATIC_URL = '/static/'
  103. STATIC_ROOT = os.path.join(BASE_DIR, 'build', 'static')
  104. # List of callables that know how to import templates from various sources.
  105. GTEMPLATE_LOADERS = (
  106. 'django.template.loaders.filesystem.Loader',
  107. 'django.template.loaders.app_directories.Loader'
  108. )
  109. MIDDLEWARE = [
  110. # The order matters
  111. 'desktop.middleware.MetricsMiddleware',
  112. 'desktop.middleware.EnsureSafeMethodMiddleware',
  113. 'desktop.middleware.AuditLoggingMiddleware',
  114. 'django.middleware.common.CommonMiddleware',
  115. 'django.contrib.sessions.middleware.SessionMiddleware',
  116. 'django.contrib.auth.middleware.AuthenticationMiddleware',
  117. 'desktop.middleware.ProxyMiddleware',
  118. 'desktop.middleware.SpnegoMiddleware',
  119. 'desktop.middleware.HueRemoteUserMiddleware',
  120. 'django.middleware.locale.LocaleMiddleware',
  121. 'django_babel.middleware.LocaleMiddleware',
  122. 'desktop.middleware.AjaxMiddleware',
  123. 'django.middleware.security.SecurityMiddleware',
  124. 'django.middleware.clickjacking.XFrameOptionsMiddleware',
  125. 'desktop.middleware.ContentSecurityPolicyMiddleware',
  126. # Must be after Session, Auth, and Ajax. Before everything else.
  127. 'desktop.middleware.LoginAndPermissionMiddleware',
  128. 'django.contrib.messages.middleware.MessageMiddleware',
  129. 'desktop.middleware.NotificationMiddleware',
  130. 'desktop.middleware.ExceptionMiddleware',
  131. 'desktop.middleware.ClusterMiddleware',
  132. 'django.middleware.csrf.CsrfViewMiddleware',
  133. 'django.middleware.http.ConditionalGetMiddleware',
  134. #@TODO@ Prakash to check FailedLoginMiddleware working or not?
  135. #'axes.middleware.FailedLoginMiddleware',
  136. 'desktop.middleware.MimeTypeJSFileFixStreamingMiddleware',
  137. 'crequest.middleware.CrequestMiddleware',
  138. ]
  139. # if os.environ.get(ENV_DESKTOP_DEBUG):
  140. # MIDDLEWARE.append('desktop.middleware.HtmlValidationMiddleware')
  141. # logging.debug("Will try to validate generated HTML.")
  142. ROOT_URLCONF = 'desktop.urls'
  143. # Hue runs its own wsgi applications
  144. WSGI_APPLICATION = None
  145. GTEMPLATE_DIRS = (
  146. get_desktop_root("core/templates"),
  147. )
  148. INSTALLED_APPS = [
  149. 'django.contrib.auth',
  150. 'django.contrib.contenttypes',
  151. 'django.contrib.sessions',
  152. 'django.contrib.sites',
  153. 'django.contrib.staticfiles',
  154. 'django.contrib.admin',
  155. 'django_extensions',
  156. # 'debug_toolbar',
  157. #'south', # database migration tool
  158. # i18n support
  159. 'django_babel',
  160. # Desktop injects all the other installed apps into here magically.
  161. 'desktop',
  162. # App that keeps track of failed logins.
  163. 'axes',
  164. 'webpack_loader',
  165. 'django_prometheus',
  166. 'crequest',
  167. #'django_celery_results',
  168. ]
  169. WEBPACK_LOADER = {
  170. 'DEFAULT': {
  171. 'BUNDLE_DIR_NAME': 'desktop/js/bundles/hue/',
  172. 'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats.json')
  173. },
  174. 'WORKERS': {
  175. 'BUNDLE_DIR_NAME': 'desktop/js/bundles/workers/',
  176. 'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats-workers.json')
  177. },
  178. 'LOGIN': {
  179. 'BUNDLE_DIR_NAME': 'desktop/js/bundles/login/',
  180. 'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats-login.json')
  181. }
  182. }
  183. LOCALE_PATHS = [
  184. get_desktop_root('core/src/desktop/locale')
  185. ]
  186. # Keep default values up to date
  187. GTEMPLATE_CONTEXT_PROCESSORS = (
  188. 'django.contrib.auth.context_processors.auth',
  189. 'django.template.context_processors.debug',
  190. 'django.template.context_processors.i18n',
  191. 'django.template.context_processors.media',
  192. 'django.template.context_processors.request',
  193. 'django.contrib.messages.context_processors.messages',
  194. # Not default
  195. 'desktop.context_processors.app_name',
  196. )
  197. TEMPLATES = [
  198. {
  199. 'BACKEND': 'djangomako.backends.MakoBackend',
  200. 'DIRS': GTEMPLATE_DIRS,
  201. 'NAME': 'mako',
  202. 'OPTIONS': {
  203. 'context_processors': GTEMPLATE_CONTEXT_PROCESSORS,
  204. 'loaders': GTEMPLATE_LOADERS,
  205. },
  206. },
  207. {
  208. 'BACKEND': 'django.template.backends.django.DjangoTemplates',
  209. 'DIRS': [
  210. get_desktop_root("core/templates/debug_toolbar"),
  211. get_desktop_root("core/templates/djangosaml2"),
  212. ],
  213. 'NAME': 'django',
  214. 'APP_DIRS': True,
  215. },
  216. ]
  217. # Desktop doesn't use an auth profile module, because
  218. # because it doesn't mesh very well with the notion
  219. # of having multiple apps. If your app needs
  220. # to store data related to users, it should
  221. # manage its own table with an appropriate foreign key.
  222. AUTH_PROFILE_MODULE = None
  223. LOGIN_REDIRECT_URL = "/"
  224. LOGOUT_REDIRECT_URL = "/" # For djangosaml2 bug.
  225. PYLINTRC = get_run_root('.pylintrc')
  226. # Custom CSRF Failure View
  227. CSRF_FAILURE_VIEW = 'desktop.views.csrf_failure'
  228. ############################################################
  229. # Part 4: Installation of apps
  230. ############################################################
  231. _config_dir = os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))
  232. # Libraries are loaded and configured before the apps
  233. appmanager.load_libs()
  234. _lib_conf_modules = [dict(module=app.conf, config_key=None) for app in appmanager.DESKTOP_LIBS if app.conf is not None]
  235. LOCALE_PATHS.extend([app.locale_path for app in appmanager.DESKTOP_LIBS])
  236. # Load desktop config
  237. _desktop_conf_modules = [dict(module=desktop.conf, config_key=None)]
  238. conf.initialize(_desktop_conf_modules, _config_dir)
  239. # Register the redaction filters into the root logger as soon as possible.
  240. desktop.redaction.register_log_filtering(desktop.conf.get_redaction_policy())
  241. # Activate l10n
  242. # Install apps
  243. appmanager.load_apps(desktop.conf.APP_BLACKLIST.get())
  244. for app in appmanager.DESKTOP_APPS:
  245. INSTALLED_APPS.extend(app.django_apps)
  246. LOCALE_PATHS.append(app.locale_path)
  247. logging.debug("Installed Django modules: %s" % ",".join(map(str, appmanager.DESKTOP_MODULES)))
  248. # Load app configuration
  249. _app_conf_modules = [dict(module=app.conf, config_key=app.config_key) for app in appmanager.DESKTOP_APPS if app.conf is not None]
  250. conf.initialize(_lib_conf_modules, _config_dir)
  251. conf.initialize(_app_conf_modules, _config_dir)
  252. # Now that we've loaded the desktop conf, set the django DEBUG mode based on the conf.
  253. DEBUG = desktop.conf.DJANGO_DEBUG_MODE.get()
  254. GTEMPLATE_DEBUG = DEBUG
  255. if DEBUG: # For simplification, force all DEBUG when django_debug_mode is True and re-apply the loggers
  256. os.environ[ENV_DESKTOP_DEBUG] = 'True'
  257. desktop.log.basic_logging(os.environ[ENV_HUE_PROCESS_NAME])
  258. desktop.log.fancy_logging()
  259. ############################################################
  260. # Part 4a: Django configuration that requires bound Desktop
  261. # configs.
  262. ############################################################
  263. if desktop.conf.ENABLE_ORGANIZATIONS.get():
  264. AUTH_USER_MODEL = 'useradmin.OrganizationUser'
  265. MIGRATION_MODULES = {
  266. 'beeswax': 'beeswax.org_migrations',
  267. 'useradmin': 'useradmin.org_migrations',
  268. 'desktop': 'desktop.org_migrations',
  269. }
  270. # Configure allowed hosts
  271. ALLOWED_HOSTS = desktop.conf.ALLOWED_HOSTS.get()
  272. X_FRAME_OPTIONS = desktop.conf.X_FRAME_OPTIONS.get()
  273. # Configure admins
  274. ADMINS = []
  275. for admin in desktop.conf.DJANGO_ADMINS.get():
  276. admin_conf = desktop.conf.DJANGO_ADMINS[admin]
  277. if 'name' in admin_conf.bind_to and 'email' in admin_conf.bind_to:
  278. ADMINS.append(((admin_conf.NAME.get(), admin_conf.EMAIL.get())))
  279. ADMINS = tuple(ADMINS)
  280. MANAGERS = ADMINS
  281. SERVER_EMAIL = desktop.conf.DJANGO_SERVER_EMAIL.get()
  282. EMAIL_BACKEND = desktop.conf.DJANGO_EMAIL_BACKEND.get()
  283. EMAIL_SUBJECT_PREFIX = 'Hue %s - ' % desktop.conf.CLUSTER_ID.get()
  284. # Configure database
  285. if os.getenv('DESKTOP_DB_CONFIG'):
  286. conn_string = os.getenv('DESKTOP_DB_CONFIG')
  287. logging.debug("DESKTOP_DB_CONFIG SET: %s" % (conn_string))
  288. default_db = dict(
  289. list(
  290. zip(["ENGINE", "NAME", "TEST_NAME", "USER", "PASSWORD", "HOST", "PORT"], conn_string.split(':'))
  291. )
  292. )
  293. default_db['NAME'] = default_db['NAME'].replace('#', ':') # For is_db_alive command
  294. else:
  295. test_name = os.environ.get('DESKTOP_DB_TEST_NAME', get_desktop_root('desktop-test.db'))
  296. logging.debug("DESKTOP_DB_TEST_NAME SET: %s" % test_name)
  297. test_user = os.environ.get('DESKTOP_DB_TEST_USER', 'hue_test')
  298. logging.debug("DESKTOP_DB_TEST_USER SET: %s" % test_user)
  299. default_db = {
  300. "ENGINE": desktop.conf.DATABASE.ENGINE.get(),
  301. "NAME": desktop.conf.DATABASE.NAME.get(),
  302. "USER": desktop.conf.DATABASE.USER.get(),
  303. "SCHEMA": desktop.conf.DATABASE.SCHEMA.get(),
  304. "PASSWORD": desktop.conf.get_database_password(),
  305. "HOST": desktop.conf.DATABASE.HOST.get(),
  306. "PORT": str(desktop.conf.DATABASE.PORT.get()),
  307. "OPTIONS": force_dict_to_strings(desktop.conf.DATABASE.OPTIONS.get()),
  308. # DB used for tests
  309. "TEST_NAME": test_name,
  310. "TEST_USER": test_user,
  311. # Wrap each request in a transaction.
  312. "ATOMIC_REQUESTS": True,
  313. "CONN_MAX_AGE": desktop.conf.DATABASE.CONN_MAX_AGE.get(),
  314. }
  315. DATABASES = {
  316. 'default': default_db
  317. }
  318. if desktop.conf.QUERY_DATABASE.HOST.get():
  319. DATABASES['query'] = {
  320. 'ENGINE': desktop.conf.QUERY_DATABASE.ENGINE.get(),
  321. 'HOST': desktop.conf.QUERY_DATABASE.HOST.get(),
  322. 'NAME': desktop.conf.QUERY_DATABASE.NAME.get(),
  323. 'USER': desktop.conf.QUERY_DATABASE.USER.get(),
  324. 'PASSWORD': desktop.conf.QUERY_DATABASE.PASSWORD.get(),
  325. 'OPTIONS': desktop.conf.QUERY_DATABASE.OPTIONS.get(),
  326. 'PORT': desktop.conf.QUERY_DATABASE.PORT.get(),
  327. "SCHEMA": desktop.conf.QUERY_DATABASE.SCHEMA.get(),
  328. }
  329. CACHES = {
  330. 'default': {
  331. 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', # TODO: Parameterize here for all the caches
  332. 'LOCATION': 'unique-hue'
  333. },
  334. 'axes_cache': {
  335. 'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
  336. },
  337. }
  338. AXES_CACHE = 'axes_cache'
  339. CACHES_HIVE_DISCOVERY_KEY = 'hive_discovery'
  340. CACHES[CACHES_HIVE_DISCOVERY_KEY] = {
  341. 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
  342. 'LOCATION': CACHES_HIVE_DISCOVERY_KEY
  343. }
  344. CACHES_CELERY_KEY = 'celery'
  345. CACHES_CELERY_QUERY_RESULT_KEY = 'celery_query_results'
  346. if desktop.conf.TASK_SERVER.ENABLED.get():
  347. CACHES[CACHES_CELERY_KEY] = json.loads(desktop.conf.TASK_SERVER.EXECUTION_STORAGE.get())
  348. if desktop.conf.TASK_SERVER.RESULT_CACHE.get():
  349. CACHES[CACHES_CELERY_QUERY_RESULT_KEY] = json.loads(desktop.conf.TASK_SERVER.RESULT_CACHE.get())
  350. # Configure sessions
  351. SESSION_COOKIE_NAME = desktop.conf.SESSION.COOKIE_NAME.get()
  352. SESSION_COOKIE_AGE = desktop.conf.SESSION.TTL.get()
  353. SESSION_COOKIE_SECURE = desktop.conf.SESSION.SECURE.get()
  354. SESSION_EXPIRE_AT_BROWSER_CLOSE = desktop.conf.SESSION.EXPIRE_AT_BROWSER_CLOSE.get()
  355. # HTTP only
  356. SESSION_COOKIE_HTTPONLY = desktop.conf.SESSION.HTTP_ONLY.get()
  357. CSRF_COOKIE_AGE = None if desktop.conf.SESSION.CSRF_COOKIE_AGE.get() == 0 else desktop.conf.SESSION.CSRF_COOKIE_AGE.get()
  358. CSRF_COOKIE_SECURE = desktop.conf.SESSION.SECURE.get()
  359. CSRF_COOKIE_HTTPONLY = desktop.conf.SESSION.HTTP_ONLY.get()
  360. CSRF_COOKIE_NAME = 'csrftoken'
  361. TRUSTED_ORIGINS = []
  362. if desktop.conf.SESSION.TRUSTED_ORIGINS.get():
  363. TRUSTED_ORIGINS += desktop.conf.SESSION.TRUSTED_ORIGINS.get()
  364. # This is required for knox
  365. if desktop.conf.KNOX.KNOX_PROXYHOSTS.get(): # The hosts provided here don't have port. Add default knox port
  366. if desktop.conf.KNOX.KNOX_PORTS.get():
  367. hostport = []
  368. ports = [ # In case the ports are in hostname
  369. host.split(':')[1] for host in desktop.conf.KNOX.KNOX_PROXYHOSTS.get() if len(host.split(':')) > 1
  370. ]
  371. for port in ports + desktop.conf.KNOX.KNOX_PORTS.get():
  372. if port == '80':
  373. port = '' # Default port needs to be empty
  374. else:
  375. port = ':' + port
  376. hostport += [host.split(':')[0] + port for host in desktop.conf.KNOX.KNOX_PROXYHOSTS.get()]
  377. TRUSTED_ORIGINS += hostport
  378. else:
  379. TRUSTED_ORIGINS += desktop.conf.KNOX.KNOX_PROXYHOSTS.get()
  380. if TRUSTED_ORIGINS:
  381. CSRF_TRUSTED_ORIGINS = TRUSTED_ORIGINS
  382. SECURE_HSTS_SECONDS = desktop.conf.SECURE_HSTS_SECONDS.get()
  383. SECURE_HSTS_INCLUDE_SUBDOMAINS = desktop.conf.SECURE_HSTS_INCLUDE_SUBDOMAINS.get()
  384. SECURE_CONTENT_TYPE_NOSNIFF = desktop.conf.SECURE_CONTENT_TYPE_NOSNIFF.get()
  385. SECURE_BROWSER_XSS_FILTER = desktop.conf.SECURE_BROWSER_XSS_FILTER.get()
  386. SECURE_SSL_REDIRECT = desktop.conf.SECURE_SSL_REDIRECT.get()
  387. SECURE_SSL_HOST = desktop.conf.SECURE_SSL_HOST.get()
  388. SECURE_REDIRECT_EXEMPT = desktop.conf.SECURE_REDIRECT_EXEMPT.get()
  389. # django-nose test specifics
  390. TEST_RUNNER = 'desktop.lib.test_runners.HueTestRunner'
  391. # Turn off cache middleware
  392. if 'test' in sys.argv:
  393. CACHE_MIDDLEWARE_SECONDS = 0
  394. # Limit Nose coverage to Hue apps
  395. NOSE_ARGS = [
  396. '--cover-package=%s' % ','.join([app.name for app in appmanager.DESKTOP_APPS + appmanager.DESKTOP_LIBS]),
  397. '--no-path-adjustment',
  398. '--traverse-namespace'
  399. ]
  400. TIME_ZONE = desktop.conf.TIME_ZONE.get()
  401. if desktop.conf.DEMO_ENABLED.get():
  402. AUTHENTICATION_BACKENDS = ('desktop.auth.backend.DemoBackend',)
  403. else:
  404. AUTHENTICATION_BACKENDS = tuple(desktop.conf.AUTH.BACKEND.get())
  405. EMAIL_HOST = desktop.conf.SMTP.HOST.get()
  406. EMAIL_PORT = desktop.conf.SMTP.PORT.get()
  407. EMAIL_HOST_USER = desktop.conf.SMTP.USER.get()
  408. EMAIL_HOST_PASSWORD = desktop.conf.get_smtp_password()
  409. EMAIL_USE_TLS = desktop.conf.SMTP.USE_TLS.get()
  410. DEFAULT_FROM_EMAIL = desktop.conf.SMTP.DEFAULT_FROM.get()
  411. if EMAIL_BACKEND == 'sendgrid_backend.SendgridBackend':
  412. SENDGRID_API_KEY = desktop.conf.get_smtp_password()
  413. SENDGRID_SANDBOX_MODE_IN_DEBUG = DEBUG
  414. if desktop.conf.has_channels():
  415. INSTALLED_APPS.append('channels')
  416. ASGI_APPLICATION = 'desktop.routing.application'
  417. CHANNEL_LAYERS = {
  418. 'default': {
  419. 'BACKEND': 'channels_redis.core.RedisChannelLayer',
  420. 'CONFIG': {
  421. 'hosts': [(desktop.conf.WEBSOCKETS.LAYER_HOST.get(), desktop.conf.WEBSOCKETS.LAYER_PORT.get())],
  422. },
  423. },
  424. }
  425. # Used for securely creating sessions. Should be unique and not shared with anybody.
  426. # Changing auth backends will invalidate all open sessions.
  427. SECRET_KEY = desktop.conf.get_secret_key()
  428. if SECRET_KEY:
  429. SECRET_KEY += str(AUTHENTICATION_BACKENDS)
  430. else:
  431. SECRET_KEY = str(uuid.uuid4())
  432. # Axes
  433. AXES_LOGIN_FAILURE_LIMIT = desktop.conf.AUTH.LOGIN_FAILURE_LIMIT.get()
  434. AXES_LOCK_OUT_AT_FAILURE = desktop.conf.AUTH.LOGIN_LOCK_OUT_AT_FAILURE.get()
  435. AXES_COOLOFF_TIME = None
  436. if desktop.conf.AUTH.LOGIN_COOLOFF_TIME.get() and desktop.conf.AUTH.LOGIN_COOLOFF_TIME.get() != 0:
  437. AXES_COOLOFF_TIME = desktop.conf.AUTH.LOGIN_COOLOFF_TIME.get()
  438. AXES_USE_USER_AGENT = desktop.conf.AUTH.LOGIN_LOCK_OUT_USE_USER_AGENT.get()
  439. AXES_LOCK_OUT_BY_COMBINATION_USER_AND_IP = desktop.conf.AUTH.LOGIN_LOCK_OUT_BY_COMBINATION_USER_AND_IP.get()
  440. AXES_BEHIND_REVERSE_PROXY = desktop.conf.AUTH.BEHIND_REVERSE_PROXY.get()
  441. AXES_REVERSE_PROXY_HEADER = desktop.conf.AUTH.REVERSE_PROXY_HEADER.get()
  442. LOGIN_URL = '/hue/accounts/login'
  443. # SAML
  444. SAML_AUTHENTICATION = 'libsaml.backend.SAML2Backend' in AUTHENTICATION_BACKENDS
  445. if SAML_AUTHENTICATION:
  446. from libsaml.saml_settings import *
  447. INSTALLED_APPS.append('libsaml')
  448. LOGIN_URL = '/saml2/login/'
  449. SESSION_EXPIRE_AT_BROWSER_CLOSE = True
  450. # Middleware classes.
  451. for middleware in desktop.conf.MIDDLEWARE.get():
  452. MIDDLEWARE.append(middleware)
  453. # OpenID Connect
  454. def is_oidc_configured():
  455. return 'desktop.auth.backend.OIDCBackend' in AUTHENTICATION_BACKENDS
  456. if is_oidc_configured():
  457. INSTALLED_APPS.append('mozilla_django_oidc')
  458. if 'desktop.auth.backend.AllowFirstUserDjangoBackend' not in AUTHENTICATION_BACKENDS:
  459. # when multi-backend auth, standard login URL '/hue/accounts/login' is used.
  460. LOGIN_URL = '/oidc/authenticate/'
  461. SESSION_EXPIRE_AT_BROWSER_CLOSE = True
  462. MIDDLEWARE.append('mozilla_django_oidc.middleware.SessionRefresh')
  463. OIDC_RENEW_ID_TOKEN_EXPIRY_SECONDS = 15 * 60
  464. OIDC_RP_SIGN_ALGO = 'RS256'
  465. OIDC_RP_CLIENT_ID = desktop.conf.OIDC.OIDC_RP_CLIENT_ID.get()
  466. OIDC_RP_CLIENT_SECRET = desktop.conf.OIDC.OIDC_RP_CLIENT_SECRET.get()
  467. OIDC_OP_AUTHORIZATION_ENDPOINT = desktop.conf.OIDC.OIDC_OP_AUTHORIZATION_ENDPOINT.get()
  468. OIDC_OP_TOKEN_ENDPOINT = desktop.conf.OIDC.OIDC_OP_TOKEN_ENDPOINT.get()
  469. OIDC_OP_USER_ENDPOINT = desktop.conf.OIDC.OIDC_OP_USER_ENDPOINT.get()
  470. OIDC_RP_IDP_SIGN_KEY = desktop.conf.OIDC.OIDC_RP_IDP_SIGN_KEY.get()
  471. OIDC_OP_JWKS_ENDPOINT = desktop.conf.OIDC.OIDC_OP_JWKS_ENDPOINT.get()
  472. OIDC_VERIFY_SSL = desktop.conf.OIDC.OIDC_VERIFY_SSL.get()
  473. LOGIN_REDIRECT_URL = desktop.conf.OIDC.LOGIN_REDIRECT_URL.get()
  474. LOGOUT_REDIRECT_URL = desktop.conf.OIDC.LOGOUT_REDIRECT_URL.get()
  475. LOGIN_REDIRECT_URL_FAILURE = desktop.conf.OIDC.LOGIN_REDIRECT_URL_FAILURE.get()
  476. OIDC_STORE_ACCESS_TOKEN = True
  477. OIDC_STORE_ID_TOKEN = True
  478. OIDC_STORE_REFRESH_TOKEN = True
  479. OIDC_CREATE_USER = desktop.conf.OIDC.CREATE_USERS_ON_LOGIN.get()
  480. OIDC_USERNAME_ATTRIBUTE = desktop.conf.OIDC.OIDC_USERNAME_ATTRIBUTE.get()
  481. # OAuth
  482. OAUTH_AUTHENTICATION = 'liboauth.backend.OAuthBackend' in AUTHENTICATION_BACKENDS
  483. if OAUTH_AUTHENTICATION:
  484. INSTALLED_APPS.append('liboauth')
  485. LOGIN_URL = '/oauth/accounts/login'
  486. SESSION_EXPIRE_AT_BROWSER_CLOSE = True
  487. # URL Redirection white list.
  488. if desktop.conf.REDIRECT_WHITELIST.get():
  489. MIDDLEWARE.append('desktop.middleware.EnsureSafeRedirectURLMiddleware')
  490. # Enable X-Forwarded-Host header if the load balancer requires it
  491. USE_X_FORWARDED_HOST = desktop.conf.USE_X_FORWARDED_HOST.get()
  492. # Support HTTPS load-balancing
  493. if desktop.conf.SECURE_PROXY_SSL_HEADER.get():
  494. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
  495. # Add last activity tracking and idle session timeout
  496. if 'useradmin' in [app.name for app in appmanager.DESKTOP_APPS]:
  497. MIDDLEWARE.append('useradmin.middleware.LastActivityMiddleware')
  498. if desktop.conf.SESSION.CONCURRENT_USER_SESSION_LIMIT.get():
  499. MIDDLEWARE.append('useradmin.middleware.ConcurrentUserSessionMiddleware')
  500. LOAD_BALANCER_COOKIE = 'ROUTEID'
  501. ################################################################
  502. # Register file upload handlers
  503. # This section must go after the desktop lib modules are loaded
  504. ################################################################
  505. # Insert our custom upload handlers
  506. file_upload_handlers = [
  507. 'hadoop.fs.upload.HDFSfileUploadHandler',
  508. 'django.core.files.uploadhandler.MemoryFileUploadHandler',
  509. 'django.core.files.uploadhandler.TemporaryFileUploadHandler',
  510. ]
  511. if is_s3_enabled():
  512. file_upload_handlers.insert(0, 'aws.s3.upload.S3FileUploadHandler')
  513. if is_abfs_enabled():
  514. file_upload_handlers.insert(0, 'azure.abfs.upload.ABFSFileUploadHandler')
  515. FILE_UPLOAD_HANDLERS = tuple(file_upload_handlers)
  516. ############################################################
  517. # Necessary for South to not fuzz with tests. Fixed in South 0.7.1
  518. SKIP_SOUTH_TESTS = True
  519. # Set up environment variable so Kerberos libraries look at our private
  520. # ticket cache
  521. os.environ['KRB5CCNAME'] = desktop.conf.KERBEROS.CCACHE_PATH.get()
  522. if not os.getenv('SERVER_SOFTWARE'):
  523. os.environ['SERVER_SOFTWARE'] = 'apache'
  524. # If Hue is configured to use a CACERTS truststore, make sure that the
  525. # REQUESTS_CA_BUNDLE is set so that we can use it when we make external requests.
  526. # This is for the REST calls made by Hue with the requests library.
  527. if desktop.conf.SSL_CACERTS.get() and os.environ.get('REQUESTS_CA_BUNDLE') is None:
  528. os.environ['REQUESTS_CA_BUNDLE'] = desktop.conf.SSL_CACERTS.get()
  529. # Preventing local build failure by not validating the default value of REQUESTS_CA_BUNDLE
  530. if os.environ.get('REQUESTS_CA_BUNDLE') and os.environ.get('REQUESTS_CA_BUNDLE') != desktop.conf.SSL_CACERTS.config.default \
  531. and not os.path.isfile(os.environ['REQUESTS_CA_BUNDLE']):
  532. raise Exception(_('SSL Certificate pointed by REQUESTS_CA_BUNDLE does not exist: %s') % os.environ['REQUESTS_CA_BUNDLE'])
  533. # Instrumentation
  534. if desktop.conf.INSTRUMENTATION.get():
  535. if sys.version_info[0] > 2:
  536. gc.set_debug(gc.DEBUG_UNCOLLECTABLE)
  537. else:
  538. gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_OBJECTS)
  539. if not desktop.conf.DATABASE_LOGGING.get():
  540. def disable_database_logging():
  541. from django.db.backends.base.base import BaseDatabaseWrapper
  542. from django.db.backends.utils import CursorWrapper
  543. BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(cursor, self)
  544. disable_database_logging()
  545. ############################################################
  546. # Searching saved documents in Oracle returns following error:
  547. # DatabaseError: ORA-06502: PL/SQL: numeric or value error: character string buffer too small
  548. # This is caused by DBMS_LOB.SUBSTR(%s, 4000) in Django framework django/db/backends/oracle/base.py
  549. # Django has a ticket for this issue but unfixed: https://code.djangoproject.com/ticket/11580.
  550. # Buffer size 4000 limit the length of field equals or less than 2000 characters.
  551. #
  552. # For performance reasons and to avoid searching in huge fields, we also truncate to a max length
  553. DOCUMENT2_SEARCH_MAX_LENGTH = 2000
  554. # To avoid performace issue, config check will display warning when Document2 over this size
  555. DOCUMENT2_MAX_ENTRIES = 100000
  556. DEBUG_TOOLBAR_PATCH_SETTINGS = False
  557. def show_toolbar(request):
  558. # Here can be used to decide if showing toolbar bases on request object:
  559. # For example, limit IP address by checking request.META['REMOTE_ADDR'], which can avoid setting INTERNAL_IPS.
  560. list_allowed_users = desktop.conf.DJANGO_DEBUG_TOOL_USERS.get()
  561. is_user_allowed = list_allowed_users[0] == '' or request.user.username in list_allowed_users
  562. return DEBUG and desktop.conf.ENABLE_DJANGO_DEBUG_TOOL.get() and is_user_allowed
  563. if DEBUG and desktop.conf.ENABLE_DJANGO_DEBUG_TOOL.get():
  564. idx = MIDDLEWARE.index('desktop.middleware.ClusterMiddleware')
  565. MIDDLEWARE.insert(idx + 1, 'debug_panel.middleware.DebugPanelMiddleware')
  566. INSTALLED_APPS += (
  567. 'debug_toolbar',
  568. 'debug_panel',
  569. )
  570. DEBUG_TOOLBAR_PANELS = [
  571. 'debug_toolbar.panels.versions.VersionsPanel',
  572. 'debug_toolbar.panels.timer.TimerPanel',
  573. 'debug_toolbar.panels.settings.SettingsPanel',
  574. 'debug_toolbar.panels.headers.HeadersPanel',
  575. 'debug_toolbar.panels.request.RequestPanel',
  576. 'debug_toolbar.panels.sql.SQLPanel',
  577. 'debug_toolbar.panels.staticfiles.StaticFilesPanel',
  578. 'debug_toolbar.panels.templates.TemplatesPanel',
  579. 'debug_toolbar.panels.cache.CachePanel',
  580. 'debug_toolbar.panels.signals.SignalsPanel',
  581. 'debug_toolbar.panels.logging.LoggingPanel',
  582. 'debug_toolbar.panels.redirects.RedirectsPanel',
  583. ]
  584. DEBUG_TOOLBAR_CONFIG = {
  585. 'JQUERY_URL': os.path.join(STATIC_ROOT, 'desktop/ext/js/jquery/jquery-2.2.4.min.js'),
  586. 'RESULTS_CACHE_SIZE': 200,
  587. 'SHOW_TOOLBAR_CALLBACK': show_toolbar
  588. }
  589. CACHES.update({
  590. 'debug-panel': {
  591. 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
  592. 'LOCATION': '/var/tmp/debug-panel-cache',
  593. 'OPTIONS': {
  594. 'MAX_ENTRIES': 10000
  595. }
  596. }
  597. })
  598. ################################################################
  599. # Celery settings
  600. ################################################################
  601. if desktop.conf.TASK_SERVER.ENABLED.get() or desktop.conf.TASK_SERVER.BEAT_ENABLED.get():
  602. CELERY_BROKER_URL = desktop.conf.TASK_SERVER.BROKER_URL.get()
  603. CELERY_ACCEPT_CONTENT = ['json']
  604. CELERY_RESULT_BACKEND = desktop.conf.TASK_SERVER.CELERY_RESULT_BACKEND.get()
  605. CELERY_TASK_SERIALIZER = 'json'
  606. CELERYD_OPTS = desktop.conf.TASK_SERVER.RESULT_CELERYD_OPTS.get()
  607. # %n will be replaced with the first part of the nodename.
  608. # CELERYD_LOG_FILE="/var/log/celery/%n%I.log"
  609. # CELERYD_PID_FILE="/var/run/celery/%n.pid"
  610. # CELERY_CREATE_DIRS = 1
  611. # CELERYD_USER = desktop.conf.SERVER_USER.get()
  612. # CELERYD_GROUP = desktop.conf.SERVER_GROUP.get()
  613. if desktop.conf.TASK_SERVER.BEAT_ENABLED.get():
  614. INSTALLED_APPS.append('django_celery_beat')
  615. INSTALLED_APPS.append('timezone_field')
  616. USE_TZ = True
  617. PROMETHEUS_EXPORT_MIGRATIONS = False # Needs to be there even when enable_prometheus is not enabled
  618. if desktop.conf.ENABLE_PROMETHEUS.get():
  619. MIDDLEWARE.insert(0, 'django_prometheus.middleware.PrometheusBeforeMiddleware')
  620. MIDDLEWARE.append('django_prometheus.middleware.PrometheusAfterMiddleware')
  621. if 'mysql' in DATABASES['default']['ENGINE']:
  622. DATABASES['default']['ENGINE'] = DATABASES['default']['ENGINE'].replace('django.db.backends', 'django_prometheus.db.backends')
  623. # enable only when use these metrics: django_cache_get_total, django_cache_hits_total, django_cache_misses_total
  624. # for name, val in list(CACHES.items()):
  625. # val['BACKEND'] = val['BACKEND'].replace('django.core.cache.backends', 'django_prometheus.cache.backends')
  626. ################################################################
  627. # OpenTracing settings
  628. ################################################################
  629. if desktop.conf.TRACING.ENABLED.get():
  630. OPENTRACING_TRACE_ALL = desktop.conf.TRACING.TRACE_ALL.get()
  631. OPENTRACING_TRACER_CALLABLE = __name__ + '.tracer'
  632. def tracer():
  633. from jaeger_client import Config
  634. config = Config(
  635. config={
  636. 'sampler': {
  637. 'type': 'const',
  638. 'param': 1,
  639. },
  640. },
  641. # metrics_factory=PrometheusMetricsFactory(namespace='hue-api'),
  642. service_name='hue-api',
  643. validate=True,
  644. )
  645. return config.initialize_tracer()
  646. OPENTRACING_TRACED_ATTRIBUTES = ['META'] # Only valid if OPENTRACING_TRACE_ALL == True
  647. if desktop.conf.TRACING.TRACE_ALL.get():
  648. MIDDLEWARE.insert(0, 'django_opentracing.OpenTracingMiddleware')
  649. MODULES_TO_PATCH = (
  650. 'django.contrib.staticfiles.storage',
  651. 'django.core.cache.backends.filebased',
  652. 'django.core.cache.utils',
  653. 'django.db.backends.utils',
  654. 'django.utils.cache',
  655. )
  656. try:
  657. import hashlib
  658. hashlib.md5()
  659. except ValueError:
  660. from desktop.monkey_patches import monkey_patch_md5
  661. monkey_patch_md5(MODULES_TO_PATCH)