Przeglądaj źródła

HUE-8824 [optimizer] Refactor optimization mode selection

Romain 5 lat temu
rodzic
commit
3cee6411ad

+ 6 - 3
desktop/conf.dist/hue.ini

@@ -2011,10 +2011,13 @@
     ## api_url=http://localhost:7180/api
 
   [[optimizer]]
-    # Type of Optimizer connector to query, e.g. optimizer, dummy
-    # interface=optimizer
+    # Mode of optimization: off, local, api.
+    ## mode=off
 
-    # Hostname to Optimizer API or compatible service.
+    # Type of Optimizer connector to use, e.g. optimizer, navopt, dummy.
+    ## interface=navopt
+
+    # Hostname of Optimizer API service.
     ## hostname=navoptapi.us-west-1.optimizer.altus.cloudera.com
 
     # The name of the key of the service.

+ 6 - 3
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1999,10 +1999,13 @@
     ## api_url=http://localhost:7180/api
 
   [[optimizer]]
-    # Type of Optimizer connector to query, e.g. optimizer, dummy
-    # interface=optimizer
+    # Mode of optimization: off, local, api.
+    ## mode=off
 
-    # Hostnameto Optimizer API or compatible service.
+    # Type of Optimizer connector to use, e.g. optimizer, navopt, dummy.
+    ## interface=navopt
+
+    # Hostname of Optimizer API service.
     ## hostname=navoptapi.us-west-1.optimizer.altus.cloudera.com
 
     # The name of the key of the service.

+ 10 - 6
desktop/core/src/desktop/js/catalog/optimizer/optimizer.js

@@ -18,13 +18,17 @@ import ApiStrategy from './apiStrategy';
 import BaseStrategy from './baseStrategy';
 
 const OPTIMIZER_STRATEGIES = {
-  api: new ApiStrategy(),
-  off: new BaseStrategy()
+  api: ApiStrategy,
+  off: BaseStrategy
 };
 
 export const getOptimizer = connector => {
-  if (connector && connector.optimizer && OPTIMIZER_STRATEGIES[connector.optimizer]) {
-    return OPTIMIZER_STRATEGIES[connector.optimizer];
-  }
-  return OPTIMIZER_STRATEGIES.off;
+  // Can remove window.OPTIMIZER_MODE and hardcoded { optimizer: 'api' } when 'connector.optimizer_mode' works.
+  const strategy =
+    connector && connector.optimizer && OPTIMIZER_STRATEGIES[window.OPTIMIZER_MODE]
+      ? OPTIMIZER_STRATEGIES[window.OPTIMIZER_MODE]
+      : OPTIMIZER_STRATEGIES.off;
+
+  // Connector should contain proper 'sourceType' so that dataCatalog.getEntry works in localStrategy
+  return new strategy(connector);
 };

+ 2 - 2
desktop/core/src/desktop/models.py

@@ -42,7 +42,7 @@ from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
 from dashboard.conf import get_engines, HAS_REPORT_ENABLED
 from kafka.conf import has_kafka
-from metadata.conf import has_optimizer
+from metadata.conf import get_optimizer_mode
 from notebook.conf import DEFAULT_LIMIT, SHOW_NOTEBOOKS, get_ordered_interpreters
 from useradmin.models import User, Group, get_organization
 from useradmin.organization import _fitered_queryset
@@ -1791,7 +1791,7 @@ class ClusterConfig(object):
           'displayName': interpreter['name'],
           'buttonName': _('Query'),
           'tooltip': _('%s Query') % interpreter['type'].title(),
-          'optimizer': 'api' if has_optimizer() else 'off',  # TODO: Change to proper values
+          'optimizer': get_optimizer_mode(),
           'page': '/editor/?type=%(type)s' % interpreter,
           'is_sql': interpreter['is_sql'],
           'is_batchable': interpreter['dialect'] in ['hive', 'impala'] or interpreter['interface'] in ['oozie', 'sqlalchemy'],

+ 2 - 1
desktop/core/src/desktop/templates/global_js_constants.mako

@@ -29,7 +29,7 @@
   from filebrowser.conf import SHOW_UPLOAD_BUTTON
   from indexer.conf import ENABLE_NEW_INDEXER
   from metadata.conf import has_catalog, has_readonly_catalog, has_optimizer, has_workload_analytics, OPTIMIZER, get_optimizer_url, \
-      get_catalog_url
+      get_catalog_url, get_optimizer_mode
   from metastore.conf import ENABLE_NEW_CREATE_TABLE
   from metastore.views import has_write_access
   from notebook.conf import ENABLE_NOTEBOOK_2, ENABLE_QUERY_ANALYSIS, ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, ENABLE_SQL_INDEXER, \
@@ -104,6 +104,7 @@
   window.HAS_READ_ONLY_CATALOG = '${ has_readonly_catalog(request.user) }' === 'True' || '${ has_write_access(request.user) }' === 'False';
 
   window.HAS_OPTIMIZER = '${ has_optimizer() }' === 'True';
+  window.OPTIMIZER_MODE = '${ get_optimizer_mode() }';
   window.OPTIMIZER_URL = '${ get_optimizer_url() }'
   window.AUTO_UPLOAD_OPTIMIZER_STATS = '${ OPTIMIZER.AUTO_UPLOAD_STATS.get() }' === 'True';
 

+ 10 - 2
desktop/libs/metadata/src/metadata/conf.py

@@ -68,6 +68,9 @@ def get_optimizer_url():
 def has_optimizer():
   return OPTIMIZER.INTERFACE.get() != 'navopt' or bool(OPTIMIZER.AUTH_KEY_ID.get())
 
+def get_optimizer_mode():
+  return has_optimizer() and OPTIMIZER.MODE.get() or 'off'
+
 def has_workload_analytics():
   # Note: unused
   return bool(ALTUS.AUTH_KEY_ID.get()) and ALTUS.HAS_WA.get()
@@ -94,14 +97,19 @@ OPTIMIZER = ConfigSection(
   key='optimizer',
   help=_t("""Configuration options for Optimizer API"""),
   members=dict(
+    MODE=Config(
+      key='mode',
+      help=_t('Mode of optimization: off, local, api.'),
+      default='off'
+    ),
     INTERFACE=Config(
       key='interface',
-      help=_t('Type of Optimizer connector to query, e.g. optimizer, dummy'),
+      help=_t('Type of optimizer connector to use, e.g. optimizer, navopt, dummy.'),
       default='navopt'
     ),
     HOSTNAME=Config(
       key='hostname',
-      help=_t('Hostname to Optimizer API or compatible service.'),
+      help=_t('Hostname of Optimizer API service.'),
       default='navoptapi.us-west-1.optimizer.altus.cloudera.com'
     ),
     AUTH_KEY_ID=Config(