Browse Source

Hue [core] Improve Logger role with uniq logger config definition

Prakash Ranade 2 years ago
parent
commit
a1e874e05f

+ 1 - 1
desktop/conf/log.conf

@@ -16,7 +16,7 @@ handlers=logfile
 level=DEBUG
 level=DEBUG
 class=handlers.SocketHandler
 class=handlers.SocketHandler
 formatter=default
 formatter=default
-args=("/tmp/hue.uds",None,)
+args=('%LOG_DIR%/hue.uds',None,)
 
 
 [formatter_default]
 [formatter_default]
 class=desktop.log.formatter.Formatter
 class=desktop.log.formatter.Formatter

+ 43 - 33
desktop/core/src/desktop/loglistener.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 import argparse
 import argparse
+import json
 import logging
 import logging
 import logging.config
 import logging.config
 import os
 import os
@@ -35,8 +36,23 @@ from socketserver import ThreadingTCPServer, StreamRequestHandler
 #   reconfiguration of logging.
 #   reconfiguration of logging.
 #
 #
 #   _listener holds the server object doing the listening
 #   _listener holds the server object doing the listening
+rt = None
 _udslistener = None
 _udslistener = None
-def udslisten(server_address='/tmp/hue.uds', verify=None):
+
+def udsstopListening():
+  """
+  Stop the listening server which was created with a call to listen().
+  """
+  global _udslistener
+  logging._acquireLock()
+  try:
+    if _udslistener:
+      _udslistener.abort = 1
+      _udslistener = None
+  finally:
+    logging._releaseLock()
+
+def udslisten(server_address='hue.uds', verify=None):
   """
   """
   Start up a socket server on the specified unix domain socket, and listen for new
   Start up a socket server on the specified unix domain socket, and listen for new
   configurations.
   configurations.
@@ -47,14 +63,13 @@ def udslisten(server_address='/tmp/hue.uds', verify=None):
     This basically logs the record using whatever logging policy is
     This basically logs the record using whatever logging policy is
     configured locally.
     configured locally.
     """
     """
-
     def handle(self):
     def handle(self):
       """
       """
       Handle multiple requests - each expected to be a 4-byte length,
       Handle multiple requests - each expected to be a 4-byte length,
       followed by the LogRecord in pickle format. Logs the record
       followed by the LogRecord in pickle format. Logs the record
       according to whatever policy is configured locally.
       according to whatever policy is configured locally.
       """
       """
-      while True:
+      while self.server.ready:
         chunk = self.connection.recv(4)
         chunk = self.connection.recv(4)
         if len(chunk) < 4:
         if len(chunk) < 4:
           break
           break
@@ -62,13 +77,9 @@ def udslisten(server_address='/tmp/hue.uds', verify=None):
         chunk = self.connection.recv(slen)
         chunk = self.connection.recv(slen)
         while len(chunk) < slen:
         while len(chunk) < slen:
           chunk = chunk + self.connection.recv(slen - len(chunk))
           chunk = chunk + self.connection.recv(slen - len(chunk))
-        obj = self.unPickle(chunk)
+        obj = pickle.loads(chunk)
         record = logging.makeLogRecord(obj)
         record = logging.makeLogRecord(obj)
         self.handleLogRecord(record)
         self.handleLogRecord(record)
-
-    def unPickle(self, data):
-      return pickle.loads(data)
-
     def handleLogRecord(self, record):
     def handleLogRecord(self, record):
       # if a name is specified, we use the named logger rather than the one
       # if a name is specified, we use the named logger rather than the one
       # implied by the record.
       # implied by the record.
@@ -85,7 +96,7 @@ def udslisten(server_address='/tmp/hue.uds', verify=None):
     """
     """
     request_queue_size = 1
     request_queue_size = 1
 
 
-    def __init__(self, server_address='/tmp/hue.uds', handler=None,
+    def __init__(self, server_address='hue.uds', handler=None,
                  ready=None, verify=None):
                  ready=None, verify=None):
       ThreadingTCPServer.__init__(self, server_address, handler)
       ThreadingTCPServer.__init__(self, server_address, handler)
       logging._acquireLock()
       logging._acquireLock()
@@ -130,7 +141,7 @@ def udslisten(server_address='/tmp/hue.uds', verify=None):
       self.ready = threading.Event()
       self.ready = threading.Event()
 
 
     def run(self):
     def run(self):
-      server = self.rcvr(server_address='/tmp/hue.uds', handler=self.hdlr,
+      server = self.rcvr(server_address=self.server_address, handler=self.hdlr,
                          ready=self.ready, verify=self.verify)
                          ready=self.ready, verify=self.verify)
       self.ready.set()
       self.ready.set()
       global _udslistener
       global _udslistener
@@ -139,28 +150,24 @@ def udslisten(server_address='/tmp/hue.uds', verify=None):
       logging._releaseLock()
       logging._releaseLock()
       server.serve_until_stopped()
       server.serve_until_stopped()
 
 
-  return Server(ConfigSocketReceiver, ConfigStreamHandler, server_address, verify)
+    def stop(self):
+      udsstopListening()
+      self.ready.clear()
 
 
-def udsstopListening():
-  """
-  Stop the listening server which was created with a call to listen().
-  """
-  global _udslistener
-  logging._acquireLock()
-  try:
-    if _udslistener:
-      _udslistener.abort = 1
-      _udslistener = None
-  finally:
-    logging._releaseLock()
+    def stopped(self):
+      return self.ready.is_set()
+
+  return Server(ConfigSocketReceiver, ConfigStreamHandler, server_address, verify)
 
 
 def argprocessing(args=[], options={}):
 def argprocessing(args=[], options={}):
-  parser = argparse.ArgumentParser(prog='runloglistener', description='What this program does', epilog='Text at the bottom of help')
-  parser.add_argument('-s', '--socket', dest='socket', action='store', default='/tmp/hue.uds')
+  parser = argparse.ArgumentParser(prog='loglistener', description='Run Log listener listening the unix domain socket.')
+  parser.add_argument('-s', '--socket', dest='socket', action='store', default='')
 
 
   opts = parser.parse_args()
   opts = parser.parse_args()
   if opts.socket:
   if opts.socket:
     options['socket'] = opts.socket
     options['socket'] = opts.socket
+  else:
+    options['socket'] = "%s/hue.uds" % (os.getenv("DESKTOP_LOG_DIR", "/var/log/hue"))
 
 
 def enable_logging(args, options):
 def enable_logging(args, options):
   CONF_RE = re.compile('%LOG_DIR%')
   CONF_RE = re.compile('%LOG_DIR%')
@@ -175,7 +182,8 @@ def enable_logging(args, options):
     sio = string_io(CONF_RE.sub(_repl, raw))
     sio = string_io(CONF_RE.sub(_repl, raw))
     logging.config.fileConfig(sio)
     logging.config.fileConfig(sio)
   root_logger = logging.getLogger()
   root_logger = logging.getLogger()
-  root_logger.info("Welcome to Hue from Listener server ")
+  root_logger.info("Starting Hue Log Listener server using socket file %s" % (options["socket"]))
+  root_logger.info("Using logging.conf file %s" % (CONF_FILE))
 
 
 class LogException(Exception):
 class LogException(Exception):
   def __init__(self, e):
   def __init__(self, e):
@@ -185,12 +193,11 @@ class LogException(Exception):
   def __str__(self):
   def __str__(self):
     return self.message
     return self.message
 
 
-rt = None
 def signal_handler(sig, frame):
 def signal_handler(sig, frame):
-  print("Received %s" % sig)
   global rt
   global rt
-  udsstopListening()
-  rt.join()
+  print("Received signal to stop log listener %s" % sig)
+  rt.stop()
+  sys.exit(1)
 
 
 def start_listener(args, options):
 def start_listener(args, options):
   global rt
   global rt
@@ -204,11 +211,14 @@ def start_listener(args, options):
   signal.signal(signal.SIGINT, signal_handler)
   signal.signal(signal.SIGINT, signal_handler)
   signal.signal(signal.SIGQUIT, signal_handler)
   signal.signal(signal.SIGQUIT, signal_handler)
   enable_logging(args, options)
   enable_logging(args, options)
-  rt = udslisten(options["socket"], verify=False)
-  rt.start()
+  rt = udslisten(server_address=options["socket"], verify=None)
+  try:
+    rt.start()
+  finally:
+    rt.stop()
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
   args = sys.argv[1:]
   args = sys.argv[1:]
   options = {}
   options = {}
   argprocessing(args=args, options=options)
   argprocessing(args=args, options=options)
-  start_listener(args, options)
+  start_listener(args, options)

+ 16 - 7
desktop/core/src/desktop/management/commands/rungunicornserver.py

@@ -39,6 +39,8 @@ GUNICORN_SERVER_HELP = r"""
   Run Hue using the Gunicorn WSGI server in asynchronous mode.
   Run Hue using the Gunicorn WSGI server in asynchronous mode.
 """
 """
 
 
+PID_FILE = None
+
 class Command(BaseCommand):
 class Command(BaseCommand):
   help = _("Gunicorn Web server for Hue.")
   help = _("Gunicorn Web server for Hue.")
 
 
@@ -68,6 +70,11 @@ def handler_app(environ, start_response):
   os.environ.setdefault("DJANGO_SETTINGS_MODULE", "desktop.settings")
   os.environ.setdefault("DJANGO_SETTINGS_MODULE", "desktop.settings")
   return get_wsgi_application()
   return get_wsgi_application()
 
 
+def post_fork(server, worker):
+  global PID_FILE
+  with open(PID_FILE, "a") as f:
+    f.write("%s\n"%worker.pid)
+
 def enable_logging(args, options):
 def enable_logging(args, options):
   HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown"
   HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown"
   # Start basic logging as soon as possible.
   # Start basic logging as soon as possible.
@@ -78,10 +85,6 @@ def enable_logging(args, options):
   desktop.log.basic_logging(os.environ["HUE_PROCESS_NAME"])
   desktop.log.basic_logging(os.environ["HUE_PROCESS_NAME"])
   logging.info("Welcome to Hue from Gunicorn server " + HUE_DESKTOP_VERSION)
   logging.info("Welcome to Hue from Gunicorn server " + HUE_DESKTOP_VERSION)
 
 
-def post_fork(server, worker):
-  with open("/tmp/gunicorn_workers.pid", "a") as f:
-    f.write("%s\n"%worker.pid)
-
 class StandaloneApplication(gunicorn.app.base.BaseApplication):
 class StandaloneApplication(gunicorn.app.base.BaseApplication):
   def __init__(self, app, options=None):
   def __init__(self, app, options=None):
     self.options = options or {}
     self.options = options or {}
@@ -112,10 +115,16 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication):
     return self.load_wsgiapp()
     return self.load_wsgiapp()
 
 
 def argprocessing(args=[], options={}):
 def argprocessing(args=[], options={}):
+  global PID_FILE
   if options['bind']:
   if options['bind']:
+    http_port = "8888"
     bind_addr = options['bind']
     bind_addr = options['bind']
+    if ":" in bind_addr:
+      http_port = bind_addr.split(":")[1]
+    PID_FILE = "/tmp/hue_%s.pid" % (http_port)
   else:
   else:
     bind_addr = conf.HTTP_HOST.get() + ":" + str(conf.HTTP_PORT.get())
     bind_addr = conf.HTTP_HOST.get() + ":" + str(conf.HTTP_PORT.get())
+    PID_FILE = "/tmp/hue_%s.pid" % (conf.HTTP_PORT.get())
   options['bind_addr'] = bind_addr
   options['bind_addr'] = bind_addr
 
 
   # Currently gunicorn does not support passphrase suppored SSL Keyfile
   # Currently gunicorn does not support passphrase suppored SSL Keyfile
@@ -161,7 +170,6 @@ def rungunicornserver(args=[], options={}):
       'limit_request_field_size': conf.LIMIT_REQUEST_FIELD_SIZE.get(),
       'limit_request_field_size': conf.LIMIT_REQUEST_FIELD_SIZE.get(),
       'limit_request_fields': conf.LIMIT_REQUEST_FIELDS.get(),
       'limit_request_fields': conf.LIMIT_REQUEST_FIELDS.get(),
       'limit_request_line': conf.LIMIT_REQUEST_LINE.get(),
       'limit_request_line': conf.LIMIT_REQUEST_LINE.get(),
-      'logconfig': '/etc/hue/conf/log.conf',
       'loglevel': 'info',
       'loglevel': 'info',
       'max_requests': 1200,                   # The maximum number of requests a worker will process before restarting.
       'max_requests': 1200,                   # The maximum number of requests a worker will process before restarting.
       'max_requests_jitter': 0,
       'max_requests_jitter': 0,
@@ -199,6 +207,7 @@ def rungunicornserver(args=[], options={}):
   StandaloneApplication(handler_app, gunicorn_options).run()
   StandaloneApplication(handler_app, gunicorn_options).run()
 
 
 def start_server(args, options):
 def start_server(args, options):
+  global PID_FILE
   argprocessing(args, options)
   argprocessing(args, options)
 
 
   # Hide the Server software version in the response body
   # Hide the Server software version in the response body
@@ -208,9 +217,9 @@ def start_server(args, options):
   # Activate django translation
   # Activate django translation
   activate_translation()
   activate_translation()
   enable_logging(args, options)
   enable_logging(args, options)
-  with open("/tmp/gunicorn_workers.pid", "w") as f:
-    f.write("%s\n"%os.getpid())
   atexit.unregister(_exit_function)
   atexit.unregister(_exit_function)
+  with open(PID_FILE, "a") as f:
+    f.write("%s\n"%os.getpid())
   rungunicornserver(args, options)
   rungunicornserver(args, options)
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':

+ 14 - 12
desktop/core/src/desktop/management/commands/testloglistener.py

@@ -26,33 +26,35 @@ from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 from django.utils.translation import gettext as _
 
 
 SERVER_HELP = r"""
 SERVER_HELP = r"""
-  Run Python log tester
+  Generate Log lines to test new Log listener functionality
 """
 """
 def argprocessing(args=[], options={}):
 def argprocessing(args=[], options={}):
-  parser = argparse.ArgumentParser(prog='testloglistener', description='What this program does', epilog='Text at the bottom of help')
-  parser.add_argument('-s', '--socket', dest='socket', action='store', default='/tmp/hue.uds')
+  parser = argparse.ArgumentParser(prog='testloglistener', description='Generate log to test new logging functionality')
+  parser.add_argument('-s', '--socket', dest='socket', action='store', default='')
 
 
   opts = parser.parse_args()
   opts = parser.parse_args()
-  if opts.socket:
+  if opts.socket != '':
     options['socket'] = opts.socket
     options['socket'] = opts.socket
+  else:
+    options['socket'] = "%s/hue.uds" % (os.getenv("DESKTOP_LOG_DIR", "/var/log/hue"))
 
 
 def enable_logging(args, options):
 def enable_logging(args, options):
   HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown"
   HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown"
   # Start basic logging as soon as possible.
   # Start basic logging as soon as possible.
-  if "HUE_PROCESS_NAME" not in os.environ:
-    _proc = os.path.basename(len(sys.argv) > 1 and sys.argv[1] or sys.argv[0])
-    os.environ["HUE_PROCESS_NAME"] = _proc
-
-  desktop.log.basic_logging(os.environ["HUE_PROCESS_NAME"])
+  desktop.log.basic_logging("rungunicornserver")
   logging.info("Welcome to Hue from Listener server " + HUE_DESKTOP_VERSION)
   logging.info("Welcome to Hue from Listener server " + HUE_DESKTOP_VERSION)
 
 
 class Command(BaseCommand):
 class Command(BaseCommand):
-  help = _("Web server for Hue.")
+  help = _("Test script for logging.")
 
 
   def add_arguments(self, parser):
   def add_arguments(self, parser):
-    parser.add_argument('--socket', help=_("Unix Domain Socket file"), action='store', default=None)
+    parser.add_argument('-s', '--socket', help=_("Unix Domain Socket file"), dest='socket',
+                        action='store', default='')
 
 
   def handle(self, *args, **options):
   def handle(self, *args, **options):
+    if options["socket"] == '':
+      options['socket'] = "%s/hue.uds" % (os.getenv("DESKTOP_LOG_DIR", "/var/log/hue"))
+
     start_testing(args, options)
     start_testing(args, options)
 
 
   def usage(self, subcommand):
   def usage(self, subcommand):
@@ -72,4 +74,4 @@ if __name__ == '__main__':
   args = sys.argv[1:]
   args = sys.argv[1:]
   options = {}
   options = {}
   argprocessing(args=args, options=options)
   argprocessing(args=args, options=options)
-  start_testing(args, options)
+  start_testing(args, options)

+ 1 - 1
tools/container/hue/hueconf/log.conf

@@ -16,7 +16,7 @@ handlers=logfile
 level=DEBUG
 level=DEBUG
 class=handlers.SocketHandler
 class=handlers.SocketHandler
 formatter=default
 formatter=default
-args=("/tmp/hue.uds",None,)
+args=('%LOG_DIR%/hue.uds',None,)
 
 
 [formatter_default]
 [formatter_default]
 class=desktop.log.formatter.Formatter
 class=desktop.log.formatter.Formatter