Przeglądaj źródła

HUE-8738 [task] Rename the server configuration options to be simpler

Romain 6 lat temu
rodzic
commit
970ae9ca70

+ 12 - 9
desktop/conf.dist/hue.ini

@@ -790,23 +790,26 @@
    # If resource intensive or blocking can be delegated to an already running task server.
    ## enabled=False
 
+   # Switch on the integration with the Task Scheduler.
+   ## beat_enabled=False
+
+   # Number of query results rows to fetch into the result storage.
+   ## fetch_result_limit=2000
+
+   # Django file storage class to use to temporarily store query results
+   ## result_storage='{"backend": "django.core.files.storage.FileSystemStorage", "properties": {"location": "./logs"}}'
+
    # How the task server and tasks communicate.
    ## broker_url=amqp://guest:guest@localhost//
 
-   # Local file system path used to store task results when using the file result backend.
-   ## result_backend=file:///$HUE_ROOT/logs
+   # Where to store task results. Defaults to local file system path. Celery comes with a several other backends.
+   ## celery_result_backend=file:///$HUE_ROOT/logs
 
    # Default options provided to the task server at startup.
    ## celeryd_opts='--time-limit=300'
 
-   # Switch on the integration with the Task Scheduler.
-   ## beat_enabled=False
-
-   # Django file storage class to use to temporarily store query results
-   ## result_file_storage='{"backend": "django.core.files.storage.FileSystemStorage", "properties": {"location": "."}}'
-
    # Django cache to use to store temporarily used data during query execution. This is in addition to result_file_storage and result_backend.
-   ## execution-storage='{"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "celery-hue"}'
+   ## execution_storage='{"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "celery-hue"}'
 
 
 ###########################################################################

+ 12 - 9
desktop/conf/pseudo-distributed.ini.tmpl

@@ -792,23 +792,26 @@
    # If resource intensive or blocking can be delegated to an already running task server.
    ## enabled=False
 
+   # Switch on the integration with the Task Scheduler.
+   ## beat_enabled=False
+
+   # Number of query results rows to fetch into the result storage.
+   ## fetch_result_limit=2000
+
+   # Django file storage class to use to temporarily store query results
+   ## result_storage='{"backend": "django.core.files.storage.FileSystemStorage", "properties": {"location": "./logs"}}'
+
    # How the task server and tasks communicate.
    ## broker_url=amqp://guest:guest@localhost//
 
-   # Local file system path used to store task results when using the file result backend.
-   ## result_backend=file://$HUE_ROOT/logs
+   # Where to store task results. Defaults to local file system path. Celery comes with a several other backends.
+   ## celery_result_backend=file:///$HUE_ROOT/logs
 
    # Default options provided to the task server at startup.
    ## celeryd_opts='--time-limit=300'
 
-   # Switch on the integration with the Task Scheduler.
-   ## beat_enabled=False
-
-   # Django file storage class to use to temporarily store query results
-   ## result_file_storage='{"backend": "django.core.files.storage.FileSystemStorage", "properties": {"location": "."}}'
-
    # Django cache to use to store temporarily used data during query execution. This is in addition to result_file_storage and result_backend.
-   ## execution-storage='{"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "celery-hue"}'
+   ## execution_storage='{"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "celery-hue"}'
 
 
 ###########################################################################

+ 9 - 9
desktop/core/src/desktop/conf.py

@@ -1665,10 +1665,10 @@ TASK_SERVER = ConfigSection(
       default='amqp://guest:guest@localhost//',
       help=_('How the task server and tasks communicate.')
     ),
-    RESULT_BACKEND = Config(
-      key='result_backend',
+    CELERY_RESULT_BACKEND = Config(
+      key='celery_result_backend',
       dynamic_default=task_server_default_result_directory,
-      help=_('Local file system path used to store task results when using the file result backend.')
+      help=_('Where to store task results. Defaults to local file system path. Celery comes with a several other backends.')
     ),
     RESULT_CELERYD_OPTS = Config(
       key='celeryd_opts',
@@ -1681,17 +1681,17 @@ TASK_SERVER = ConfigSection(
       type=coerce_bool,
       help=_('Switch on the integration with the Task Scheduler.')
     ),
-    PREFETCH_RESULT_COUNT = Config(
-      key='prefetch_result_count',
+    FETCH_RESULT_LIMIT = Config(
+      key='fetch_result_limit',
       default=2000,
       type=coerce_positive_integer,
-      help=_('Number of rows to prefetch to Hue storage')
+      help=_('Number of query results rows to fetch into the result storage.')
     ),
-    RESULT_FILE_STORAGE = Config(
-      key='result_file_storage',
+    RESULT_STORAGE = Config(
+      key='result_storage',
       type=str,
       help=_('Django file storage class to use to temporarily store query results'),
-      default='{}'
+      default='{"backend": "django.core.files.storage.FileSystemStorage", "properties": {"location": "./logs"}}'
     ),
     EXECUTION_STORAGE = Config(
       key='execution_storage',

+ 1 - 1
desktop/core/src/desktop/settings.py

@@ -663,7 +663,7 @@ if desktop.conf.TASK_SERVER.ENABLED.get():
   CELERY_BROKER_URL = desktop.conf.TASK_SERVER.BROKER_URL.get()
 
   CELERY_ACCEPT_CONTENT = ['json']
-  CELERY_RESULT_BACKEND = desktop.conf.TASK_SERVER.RESULT_BACKEND.get()
+  CELERY_RESULT_BACKEND = desktop.conf.TASK_SERVER.CELERY_RESULT_BACKEND.get()
   CELERY_TASK_SERIALIZER = 'json'
 
   CELERYD_OPTS = desktop.conf.TASK_SERVER.RESULT_CELERYD_OPTS.get()

+ 3 - 3
desktop/libs/notebook/src/notebook/tasks.py

@@ -57,7 +57,7 @@ STATE_MAP = {
   states.REJECTED: 'rejected',
   states.IGNORED: 'ignored'
 }
-storage_info = json.loads(TASK_SERVER.RESULT_FILE_STORAGE.get())
+storage_info = json.loads(TASK_SERVER.RESULT_STORAGE.get())
 storage = get_storage_class(storage_info.get('backend'))(**storage_info.get('properties', {}))
 
 class ExecutionWrapperCallback(object):
@@ -97,7 +97,7 @@ def download_to_file(notebook, snippet, file_format='csv', max_rows=-1, **kwargs
 
   with storage.open(_log_key(notebook), 'wb') as f_log:
     result_wrapper = ExecutionWrapper(api, notebook, snippet, ExecutionWrapperCallback(notebook['uuid'], meta, f_log))
-    content_generator = data_export.DataAdapter(result_wrapper, max_rows=max_rows, store_data_type_in_header=True) #TODO: Move PREFETCH_RESULT_COUNT to front end
+    content_generator = data_export.DataAdapter(result_wrapper, max_rows=max_rows, store_data_type_in_header=True) #TODO: Move FETCH_RESULT_LIMIT to front end
     response = export_csvxls.create_generator(content_generator, file_format)
 
     with storage.open(_result_key(notebook), 'wb') as f:
@@ -143,7 +143,7 @@ def _patch_status(notebook):
 def execute(*args, **kwargs):
   notebook = args[0]
   snippet = args[1]
-  kwargs['max_rows'] = TASK_SERVER.PREFETCH_RESULT_COUNT.get()
+  kwargs['max_rows'] = TASK_SERVER.FETCH_RESULT_LIMIT.get()
   _patch_status(notebook)
   download_to_file.apply_async(args=args, kwargs=kwargs, task_id=notebook['uuid'])