Browse Source

Hue [py3] Enable gunicorn based fronend server (#2739)

* Hue [py3] Enable gunicorn based fronend server

(cherry picked from commit 11001d59838c2676e9f92d56860ffb6bcd4b854d)

* Hue [py3] Enable gunicorn based fronend server
Prakash Ranade 3 years ago
parent
commit
a3c3c7f467

+ 20 - 2
desktop/conf/log.conf

@@ -46,12 +46,30 @@ level=ERROR
 handlers=errorlog
 qualname=boto
 
+[logger_gunicorn.error]
+level=INFO
+handlers=errorlog
+propagate=1
+qualname=gunicorn.error
+
+[logger_gunicorn.access]
+level=INFO
+handlers=accesslog
+propagate=0
+qualname=gunicorn.access
+
 [handler_stderr]
 class=StreamHandler
 formatter=default
 level=DEBUG
 args=(sys.stderr,)
 
+[handler_stdout]
+class=StreamHandler
+formatter=default
+level=DEBUG
+args=(sys.stdout,)
+
 [handler_accesslog]
 class=handlers.RotatingFileHandler
 level=INFO
@@ -88,10 +106,10 @@ datefmt=%d/%b/%Y %H:%M:%S %z
 ########################################
 
 [loggers]
-keys=root,access,django_auth_ldap,kazoo_client,requests_packages_urllib3_connectionpool,djangosaml2,django_db,boto
+keys=root,access,django_auth_ldap,kazoo_client,requests_packages_urllib3_connectionpool,djangosaml2,django_db,boto,gunicorn.error,gunicorn.access
 
 [handlers]
-keys=stderr,logfile,accesslog,errorlog
+keys=stderr,logfile,accesslog,errorlog,stdout
 
 [formatters]
 keys=default,access

+ 8 - 11
desktop/core/src/desktop/management/commands/rungunicornserver.py

@@ -19,6 +19,7 @@ from __future__ import unicode_literals
 
 import os
 import sys
+import ssl
 import multiprocessing
 
 import gunicorn.app.base
@@ -39,7 +40,6 @@ GUNICORN_SERVER_HELP = r"""
   Run Hue using the Gunicorn WSGI server in asynchronous mode.
 """
 
-
 class Command(BaseCommand):
   help = _("Gunicorn Web server for Hue.")
 
@@ -88,17 +88,15 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication):
   def load(self):
     return self.load_wsgiapp()
 
-
 def rungunicornserver():
   bind_addr = conf.HTTP_HOST.get() + ":" + str(conf.HTTP_PORT.get())
 
   options = {
-      'access_log_format': None,
-      'accesslog': '-',
+      'accesslog': "-",
       'backlog': None,
       'bind': [bind_addr],
       'ca_certs': conf.SSL_CACERTS.get(),     # CA certificates file
-      'capture_output': None,
+      'capture_output': True,
       'cert_reqs': None,                      # Whether client certificate is required (see stdlib ssl module)
       'certfile': conf.SSL_CERTIFICATE.get(), # SSL certificate file
       'chdir': None,
@@ -108,7 +106,7 @@ def rungunicornserver():
       'daemon': None,
       'do_handshake_on_connect': None,        # Whether to perform SSL handshake on socket connect (see stdlib ssl module)
       'enable_stdio_inheritance': None,
-      'errorlog': '-',
+      'errorlog': "-",
       'forwarded_allow_ips': None,
       'graceful_timeout': None,
       'group': None,
@@ -119,14 +117,13 @@ def rungunicornserver():
       'limit_request_fields': None,
       'limit_request_line': None,
       'logconfig': None,
-      'logger_class': None,
-      'loglevel': 'debug',
+      'loglevel': 'info',
       'max_requests': None,
       'max_requests_jitter': None,
       'paste': None,
       'pidfile': None,
       'preload_app': None,
-      'proc_name': None,
+      'proc_name': "hue",
       'proxy_allow_ips': None,
       'proxy_protocol': None,
       'pythonpath': None,
@@ -136,7 +133,7 @@ def rungunicornserver():
       'reload_engine': None,
       'sendfile': None,
       'spew': None,
-      'ssl_version': None,                    # SSL version to use (see stdlib ssl module) [ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1]
+      'ssl_version': ssl.PROTOCOL_TLSv1_2,    # SSL version to use (see stdlib ssl module) [ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1]
       'statsd_host': None,
       'statsd_prefix': None,
       'suppress_ragged_eofs': None,           # Suppress ragged EOFs (see stdlib ssl module)
@@ -156,4 +153,4 @@ def rungunicornserver():
   StandaloneApplication(handler_app, options).run()
 
 if __name__ == '__main__':
-    rungunicornserver()
+  rungunicornserver()

+ 2 - 2
tools/container/hue/hue.sh

@@ -6,7 +6,7 @@ set -x
 # Time marker for both stderr and stdout
 date; date 1>&2
 
-export DESKTOP_LOG_DIR="/var/log/hive"
+export DESKTOP_LOG_DIR="/opt/hive/logs"
 export PYTHON_EGG_CACHE=$HUE_CONF_DIR/.python-eggs
 export SERVER_SOFTWARE="apache"
 
@@ -94,7 +94,7 @@ if [[ $1 == kt_renewer ]]; then
     KINIT_PATH=${KINIT_PATH-/usr/bin/kinit}
     $HUE_BIN/hue kt_renewer
   fi
-elif [[ $1 == runcpserver ]]; then
+elif [[ $1 == rungunicornserver ]]; then
   if [ -e "/etc/hue/conf/saml.ini" ]; then
     set_samlcert
   fi

+ 2 - 1
tools/container/hue/hueconf/hue.ini

@@ -1,13 +1,14 @@
 [desktop]
 allowed_hosts="*"
 use_x_forwarded_host=true
+gunicorn_number_of_workers=32
 secret_key_script=/bin/echo cloudera
 cluster_id=d6cb3d9f-953b-46ff-9dc4-ab2a9331f0db
 app_blacklist=filebrowser,oozie,search,hbase,security,metastore,pig,sqoop,spark
 time_zone=America/Los_Angeles
 django_debug_mode=false
 http_500_debug_mode=false
-cherrypy_server_threads=5
+cherrypy_server_threads=2
 default_site_encoding=utf
 collect_usage=true
 audit_event_log_dir=/etc/hue/conf

+ 83 - 12
tools/container/hue/hueconf/log.conf

@@ -1,44 +1,115 @@
-# Just log to stdout for DWX environment
+########################################
+# Definition for the different objects
+# - FOR DEVELOPMENT ONLY -
+#
+# Directories where log files are kept must already exist.
+# That's why we pick /tmp.
+#
+# The loggers are configured to write to the log files ONLY.
+# Developers may set the DESKTOP_DEBUG environment variable to
+# enable stderr logging output.
+########################################
+
 [logger_root]
-handlers=stdout
+handlers=logfile,errorlog
 
 [logger_access]
-handlers=stdout
+handlers=accesslog
 qualname=access
 
 [logger_django_auth_ldap]
-handlers=stdout
+handlers=accesslog
 qualname=django_auth_ldap
 
 [logger_kazoo_client]
-handlers=stdout
+level=INFO
+handlers=errorlog
 qualname=kazoo.client
 
 [logger_djangosaml2]
-handlers=stdout
+level=INFO
+handlers=errorlog
 qualname=djangosaml2
 
+[logger_requests_packages_urllib3_connectionpool]
+level=DEBUG
+handlers=errorlog
+qualname=requests.packages.urllib3.connectionpool
+
 [logger_django_db]
-handlers=stdout
+level=DEBUG
+handlers=errorlog
 qualname=django.db.backends
 
-# Handlers
-[handler_stdout]
+[logger_boto]
+level=ERROR
+handlers=errorlog
+qualname=boto
+
+[logger_gunicorn.error]
 level=INFO
+handlers=errorlog
+propagate=1
+qualname=gunicorn.error
+
+[logger_gunicorn.access]
+level=INFO
+handlers=accesslog
+propagate=0
+qualname=gunicorn.access
+
+[handler_stderr]
+class=StreamHandler
+formatter=default
+level=DEBUG
+args=(sys.stderr,)
+
+[handler_stdout]
 class=StreamHandler
 formatter=default
+level=DEBUG
 args=(sys.stdout,)
 
+[handler_accesslog]
+class=handlers.RotatingFileHandler
+level=INFO
+propagate=True
+formatter=access
+args=('%LOG_DIR%/access.log', 'a', 1000000, 3)
+
+[handler_errorlog]
+class=handlers.RotatingFileHandler
+level=ERROR
+formatter=default
+args=('%LOG_DIR%/error.log', 'a', 1000000, 3)
+
+[handler_logfile]
+class=handlers.RotatingFileHandler
+# Choices are DEBUG, INFO, WARNING, ERROR, CRITICAL
+level=DEBUG
+formatter=default
+args=('%LOG_DIR%/%PROC_NAME%.log', 'a', 1000000, 3)
+
 [formatter_default]
 class=desktop.log.formatter.Formatter
 format=[%(asctime)s] %(module)-12s %(levelname)-8s %(message)s
 datefmt=%d/%b/%Y %H:%M:%S %z
 
+[formatter_access]
+class=desktop.log.formatter.Formatter
+format=[%(asctime)s] %(levelname)-8s %(message)s
+datefmt=%d/%b/%Y %H:%M:%S %z
+
+
+########################################
+# A summary of loggers, handlers and formatters
+########################################
+
 [loggers]
-keys=root,access,django_auth_ldap,kazoo_client,djangosaml2,django_db
+keys=root,access,django_auth_ldap,kazoo_client,requests_packages_urllib3_connectionpool,djangosaml2,django_db,boto,gunicorn.error,gunicorn.access
 
 [handlers]
-keys=stdout
+keys=stderr,logfile,accesslog,errorlog,stdout
 
 [formatters]
-keys=default
+keys=default,access

+ 2 - 2
tools/container/hue/supervisor-files/hue_server_template

@@ -1,13 +1,13 @@
 [program:hue_server]
 directory=${HUE_HOME}
-command=${HUE_HOME}/hue.sh runcpserver
+command=${HUE_HOME}/hue.sh rungunicornserver
 autostart=true
 startretries=3
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
-environment=DESKTOP_LOG_DIR="/var/log/${HUEUSER}",PYTHON_EGG_CACHE="${HUE_CONF_DIR}/.python-eggs",SERVER_SOFTWARE="apache"
+environment=DESKTOP_LOG_DIR="/var/log/${HUEUSER}",PYTHON_EGG_CACHE="${HUE_CONF_DIR}/.python-eggs",SERVER_SOFTWARE="apache",HUE_PROCESS_NAME="hue"
 user=${HUEUSER}
 group=${HUEUSER}
 exitcodes=0,2