Browse Source

HUE-3797 [scheduler] Moving beat schedules to a submodule config

Romain 6 years ago
parent
commit
aaa03af862
3 changed files with 40 additions and 25 deletions
  1. 27 10
      desktop/conf.dist/hue.ini
  2. 7 15
      desktop/core/src/desktop/celery.py
  3. 6 0
      desktop/core/src/desktop/conf.py

+ 27 - 10
desktop/conf.dist/hue.ini

@@ -821,6 +821,23 @@
    # Django cache to use to store temporarily used data during query execution. This is in addition to result_file_storage and result_backend.
    # Django cache to use to store temporarily used data during query execution. This is in addition to result_file_storage and result_backend.
    ## execution_storage='{"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "celery-hue"}'
    ## execution_storage='{"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "celery-hue"}'
 
 
+   # Path to a file containing a list of beat schedules.
+   # beat_schedules_file=/etc/hue/conf/schedules.py
+   # e.g.
+   # periodic_tasks = [{
+   #   'add-every-monday-morning': {
+   #       'task': 'desktop.celery.debug_task',
+   #       'schedule': crontab(minute='*'),
+   #     },
+   #   },
+   #   {
+   #     'customer_count_query': {
+   #       'task': 'notebook.tasks.run_sync_query',
+   #       'schedule': crontab(minute='*'),
+   #       'args': (None, None),
+   #     },
+   #   }
+   # ]
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure the snippets available in the Notebook
 # Settings to configure the snippets available in the Notebook
@@ -1122,31 +1139,31 @@
 
 
   # Host where LLAP is running
   # Host where LLAP is running
   ## llap_server_host = localhost
   ## llap_server_host = localhost
-  
+
   # Port where LLAP Thrift server runs on
   # Port where LLAP Thrift server runs on
   ## llap_server_port = 10501
   ## llap_server_port = 10501
-  
+
   # Alternatively, use Service Discovery for LLAP (Hive Server Interactive) and/or Hiveserver2, this will override the above
   # Alternatively, use Service Discovery for LLAP (Hive Server Interactive) and/or Hiveserver2, this will override the above
-  
-  # Whether to use Service Discovery for LLAP 
+
+  # Whether to use Service Discovery for LLAP
   ## hive_discovery_llap = true
   ## hive_discovery_llap = true
-  
+
   # is llap (hive server interactive) running in an HA configuration (more than 1)
   # is llap (hive server interactive) running in an HA configuration (more than 1)
   # important as the zookeeper structure is different
   # important as the zookeeper structure is different
   ## hive_discovery_llap_ha = false
   ## hive_discovery_llap_ha = false
-  
+
   # Shortcuts to finding LLAP znode Key
   # Shortcuts to finding LLAP znode Key
   # Non-HA - hiveserver-interactive-site - hive.server2.zookeeper.namespace ex hive2 = /hive2
   # Non-HA - hiveserver-interactive-site - hive.server2.zookeeper.namespace ex hive2 = /hive2
   # HA-NonKerberized - <llap_app_name>_llap ex app name llap0 = /llap0_llap
   # HA-NonKerberized - <llap_app_name>_llap ex app name llap0 = /llap0_llap
   # HA-Kerberized - <llap_app_name>_llap-sasl ex app name llap0 = /llap0_llap-sasl
   # HA-Kerberized - <llap_app_name>_llap-sasl ex app name llap0 = /llap0_llap-sasl
   ## hive_discovery_llap_znode = /hiveserver2-hive2
   ## hive_discovery_llap_znode = /hiveserver2-hive2
-  
+
   # Whether to use Service Discovery for HiveServer2
   # Whether to use Service Discovery for HiveServer2
   ## hive_discovery_hs2 = true
   ## hive_discovery_hs2 = true
-  
+
   # Hiveserver2 is hive-site hive.server2.zookeeper.namespace ex hiveserver2 = /hiverserver2
   # Hiveserver2 is hive-site hive.server2.zookeeper.namespace ex hiveserver2 = /hiverserver2
   ## hive_discovery_hiveserver2_znode = /hiveserver2
   ## hive_discovery_hiveserver2_znode = /hiveserver2
-  
+
   # Applicable only for LLAP HA
   # Applicable only for LLAP HA
   # To keep the load on zookeeper to a minimum
   # To keep the load on zookeeper to a minimum
   # ---- we cache the LLAP activeEndpoint for the cache_timeout period
   # ---- we cache the LLAP activeEndpoint for the cache_timeout period
@@ -1155,7 +1172,7 @@
   ## caches_hive_discovery_key = hive_discovery
   ## caches_hive_discovery_key = hive_discovery
   ## caches_hive_discovery_key_path = /tmp/hive_discovery_cache
   ## caches_hive_discovery_key_path = /tmp/hive_discovery_cache
   ## cache_timeout = 60
   ## cache_timeout = 60
-  
+
   # Host where Hive Metastore Server (HMS) is running.
   # Host where Hive Metastore Server (HMS) is running.
   # If Kerberos security is enabled, the fully-qualified domain name (FQDN) is required.
   # If Kerberos security is enabled, the fully-qualified domain name (FQDN) is required.
   ## hive_metastore_host=localhost
   ## hive_metastore_host=localhost

+ 7 - 15
desktop/core/src/desktop/celery.py

@@ -17,11 +17,13 @@
 
 
 from __future__ import absolute_import, unicode_literals
 from __future__ import absolute_import, unicode_literals
 
 
+import imp
 import os
 import os
 
 
 from celery import Celery
 from celery import Celery
 from celery.schedules import crontab
 from celery.schedules import crontab
 
 
+from desktop.conf import TASK_SERVER
 from desktop.settings import TIME_ZONE, INSTALLED_APPS
 from desktop.settings import TIME_ZONE, INSTALLED_APPS
 
 
 
 
@@ -50,18 +52,8 @@ def debug_task(self):
 if 'django_celery_beat' in INSTALLED_APPS:
 if 'django_celery_beat' in INSTALLED_APPS:
   app.conf.beat_schedule = {}
   app.conf.beat_schedule = {}
 
 
-  if True:
-    app.conf.beat_schedule.update({
-      'add-every-monday-morning': {
-        'task': 'desktop.celery.debug_task',
-        'schedule': crontab(minute='*'),
-        #'args': (16, 16),
-      },
-    })
-    app.conf.beat_schedule.update({
-      'customer_count_query': {
-        'task': 'notebook.tasks.run_sync_query',
-        'schedule': crontab(minute='*'),
-        'args': (None, None),
-      },
-    })
+  if TASK_SERVER.BEAT_SCHEDULES_FILE.get():
+    schedules = imp.load_source('schedules', TASK_SERVER.BEAT_SCHEDULES_FILE.get())
+
+    for schedule in schedules.periodic_tasks:
+      app.conf.beat_schedule.update(schedule)

+ 6 - 0
desktop/core/src/desktop/conf.py

@@ -1699,6 +1699,12 @@ TASK_SERVER = ConfigSection(
       type=coerce_bool,
       type=coerce_bool,
       help=_('Switch on the integration with the Task Scheduler.')
       help=_('Switch on the integration with the Task Scheduler.')
     ),
     ),
+    BEAT_SCHEDULES_FILE = Config(
+      key='beat_schedules_file',
+      default='',
+      type=str,
+      help=_('Path to a file containing a list of beat schedules.')
+    ),
     FETCH_RESULT_LIMIT = Config(
     FETCH_RESULT_LIMIT = Config(
       key='fetch_result_limit',
       key='fetch_result_limit',
       default=2000,
       default=2000,