Bläddra i källkod

HUE-8824 [optimizer] Revamp the dummy Client to support topTables

Romain 6 år sedan
förälder
incheckning
c99b471758

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1985,6 +1985,9 @@
     ## api_url=http://localhost:7180/api
 
   [[optimizer]]
+    # Type of Optimizer connector to query, e.g. optimizer, dummy
+    # interface=optimizer
+
     # Hostname to Optimizer API or compatible service.
     ## hostname=navoptapi.us-west-1.optimizer.altus.cloudera.com
 

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1973,6 +1973,9 @@
     ## api_url=http://localhost:7180/api
 
   [[optimizer]]
+    # Type of Optimizer connector to query, e.g. optimizer, dummy
+    # interface=optimizer
+
     # Hostnameto Optimizer API or compatible service.
     ## hostname=navoptapi.us-west-1.optimizer.altus.cloudera.com
 

+ 1 - 1
desktop/libs/metadata/src/metadata/conf.py

@@ -96,7 +96,7 @@ OPTIMIZER = ConfigSection(
   members=dict(
     INTERFACE=Config(
       key='interface',
-      help=_t('Type of Optimizer to connect to, e.g. Navigator Optimizer...'),
+      help=_t('Type of Optimizer connector to query, e.g. optimizer, dummy'),
       default='optimizer'),
     HOSTNAME=Config(
       key='hostname',

+ 16 - 0
desktop/libs/metadata/src/metadata/optimizer/base.py

@@ -80,3 +80,19 @@ class Api(object):
     self.user = user
 
   # To implement
+
+
+def _get_table_name(path):
+  column = None
+
+  if path.count('.') == 1:
+    database, table = path.split('.', 1)
+  elif path.count('.') == 2:
+    database, table, column = path.split('.', 2)
+  else:
+    database, table = 'default', path
+
+  name = {'database': database, 'table': table}
+  if column:
+    name['column'] = column
+  return name

+ 31 - 85
desktop/libs/metadata/src/metadata/optimizer/dummy_client.py

@@ -23,22 +23,11 @@ import os
 import time
 import uuid
 
-from tempfile import NamedTemporaryFile
-
-from django.core.cache import cache
-from django.utils.functional import wraps
 from django.utils.translation import ugettext as _
 
-from desktop.auth.backend import is_admin
 from desktop.lib.exceptions_renderable import PopupException
-from desktop.lib import export_csvxls
-from desktop.lib.i18n import smart_unicode
-from desktop.lib.rest.http_client import RestException
-from libsentry.sentry_site import get_hive_sentry_provider
-from libsentry.privilege_checker import get_checker, MissingSentryPrivilegeException
 
-from metadata.conf import OPTIMIZER, get_optimizer_url
-from metadata.optimizer.base import Api
+from metadata.optimizer.base import Api, _get_table_name
 
 
 LOG = logging.getLogger(__name__)
@@ -50,13 +39,6 @@ class DummyClient(Api):
 
   def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id=None):
     self.user = user
-    self._api_url = (api_url or get_optimizer_url()).strip('/')
-    self._auth_key = auth_key if auth_key else OPTIMIZER.AUTH_KEY_ID.get()
-    self._auth_key_secret = auth_key_secret if auth_key_secret else (OPTIMIZER.AUTH_KEY_SECRET.get() and OPTIMIZER.AUTH_KEY_SECRET.get().replace('\\n', '\n'))
-
-    self._api = None
-
-    self._tenant_id = tenant_id
 
 
   def get_tenant(self, cluster_id='default'):
@@ -70,14 +52,40 @@ class DummyClient(Api):
   def upload_status(self, workload_id):
     pass
 
-  # Sentry permissions work bottom to top.
-  # @check_privileges
+
   def top_tables(self, workfloadId=None, database_name='default', page_size=1000, startingToken=None):
-    data = {'results': []}
+    data = {
+      'results': [{
+          "database": "default",
+          "name": "sample_07",
+          "workloadPercent": 42,
+          "columnCount": 4,
+          "total": 6
+        }, {
+          "database": "default",
+          "name": "sample_08",
+          "workloadPercent": 21,
+          "columnCount": 4,
+          "total": 3
+        }, {
+          "database": "default",
+          "name": "web_logs",
+          "workloadPercent": 21,
+          "columnCount": 1,
+          "total": 3
+        }, {
+          "database": "default",
+          "name": "customers",
+          "workloadPercent": 7,
+          "columnCount": 44,
+          "total": 1
+        }
+      ]
+    }
 
     return data
 
-  @check_privileges
+
   def table_details(self, database_name, table_name, page_size=100, startingToken=None):
     return {}
 
@@ -97,28 +105,24 @@ class DummyClient(Api):
     raise PopupException(_('Call not supported'))
 
 
-  @check_privileges
   def top_filters(self, db_tables=None, page_size=100, startingToken=None):
     results = {'result': []}
 
     return results
 
 
-  @check_privileges
   def top_aggs(self, db_tables=None, page_size=100, startingToken=None):
     results = {'result': []}
 
     return results
 
 
-  @check_privileges
   def top_columns(self, db_tables=None, page_size=100, startingToken=None):
     results = {'results': []}
 
     return results
 
 
-  @check_privileges
   def top_joins(self, db_tables=None, page_size=100, startingToken=None):
     results = {'results': []}
 
@@ -138,61 +142,3 @@ class DummyClient(Api):
       data['results'] = list(_secure_results(data['results'], self.user))
 
     return data
-
-
-def OptimizerQueryDataAdapter(data):
-  headers = ['SQL_ID', 'ELAPSED_TIME', 'SQL_FULLTEXT', 'DATABASE']
-
-  if data and len(data[0]) == 4:
-    rows = data
-  else:
-    rows = ([str(uuid.uuid4()), 0.0, q, 'default'] for q in data)
-
-  yield headers, rows
-
-
-def _get_table_name(path):
-  column = None
-
-  if path.count('.') == 1:
-    database, table = path.split('.', 1)
-  elif path.count('.') == 2:
-    database, table, column = path.split('.', 2)
-  else:
-    database, table = 'default', path
-
-  name = {'database': database, 'table': table}
-  if column:
-    name['column'] = column
-  return name
-
-
-def _secure_results(results, user, action='SELECT'):
-    if OPTIMIZER.APPLY_SENTRY_PERMISSIONS.get():
-      checker = get_checker(user=user)
-
-      def getkey(result):
-        key = {'server': get_hive_sentry_provider()}
-
-        if 'dbName' in result:
-          key['db'] = result['dbName']
-        elif 'database' in result:
-          key['db'] = result['database']
-        if 'tableName' in result:
-          key['table'] = result['tableName']
-        elif 'table' in result:
-          key['table'] = result['table']
-        if 'columnName' in result:
-          key['column'] = result['columnName']
-        elif 'column' in result:
-          key['column'] = result['column']
-
-        return key
-
-      return checker.filter_objects(results, action, key=getkey)
-    else:
-      return results
-
-
-def _clean_query(query):
-  return ' '.join([line for line in query.strip().splitlines() if not line.strip().startswith('--')])

+ 0 - 3
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -124,14 +124,11 @@ def top_tables(request):
 
 
   tables = [{
-      'eid': table['eid'],
       'database': _get_table_name(table['name'])['database'],
       'name': _get_table_name(table['name'])['table'],
       'popularity': table['workloadPercent'],
       'column_count': table['columnCount'],
-      'patternCount': table['patternCount'],
       'total': table['total'],
-      'is_fact': table['type'] != 'Dimension'
     } for table in data['results']
   ]