Prechádzať zdrojové kódy

HUE-5712 [metadata] Upload table and column stats in the new json format

Romain Rigaux 8 rokov pred
rodič
commit
4bef9a69a0

+ 23 - 4
desktop/core/src/desktop/templates/assist.mako

@@ -1825,18 +1825,30 @@ from notebook.conf import ENABLE_QUERY_BUILDER
 
 
   <script type="text/html" id="assistant-panel-template">
   <script type="text/html" id="assistant-panel-template">
     ${ _('Tables') }
     ${ _('Tables') }
+    <!-- ko if: HAS_OPTIMIZER -->
+      <a href="javascript:void(0)" data-bind="visible: activeTables().length > 0, click: function() { huePubSub.publish('editor.table.stats.upload', activeTables()); }" title="${ _('Load table and columns stats in order to improve recommendations') }">
+        <i class="fa fa-fw fa-cloud-upload"></i>
+      </a>
+    <!-- /ko -->
     <br/>
     <br/>
     <ul data-bind="foreach: activeTables">
     <ul data-bind="foreach: activeTables">
-      <li><span data-bind="text: $data"></span> <i class="fa fa-info"></i>
+      <li>
+        <span data-bind="text: $data"></span> <i class="fa fa-info"></i>
+      </i>
     </ul>
     </ul>
 
 
     <form class="form-horizontal">
     <form class="form-horizontal">
       <fieldset>
       <fieldset>
         ${ _('Fields') }<br/>
         ${ _('Fields') }<br/>
         <ul>
         <ul>
-          <li>F1</li>
-          <li>F2</li>
-          <li>F3</li>
+          <li>'country-code' is a popular field <a href="javascript:void(0)">add</a></li>
+          <li>'gender' would be a good dimension with low cardinality (2) <a href="javascript:void(0)">add</a></li>
+          <li>'ts_s=17Q1' is the latest partition <a href="javascript:void(0)">add</a></li>
+          <li>'f1'</li>
+          <li>'f2'</li>
+          <li>'f3'</li>
+          <li>'f4'</li>
+          <li>'f5'</li>
         </ul>
         </ul>
       </fieldset>
       </fieldset>
     </form>
     </form>
@@ -1848,6 +1860,13 @@ from notebook.conf import ENABLE_QUERY_BUILDER
           <li>Popular fields for the tables are: [code, salary, amount]</li>
           <li>Popular fields for the tables are: [code, salary, amount]</li>
           <li>The query would run 2x faster by adding a WHERE date_f > '2017-01-01'</li>
           <li>The query would run 2x faster by adding a WHERE date_f > '2017-01-01'</li>
           <li>Parameterize the query?</li>
           <li>Parameterize the query?</li>
+          <li>Could be automated with integrated scheduler</li>
+          <li>Data has not been refreshed since last run 3 days ago  <i class="fa fa-warning"></i> <i class="fa fa-refresh"></i></li></li>
+          <li>A schema change happened last week, a new column 'salary_med' was added</li>
+          <li>Data statistics are not accurate, click to refresh them</li>
+          <li>Query ran 17 times last week</li>
+          <li>The datasets are sometimes joined with table [Population]</li>
+          <li>Query would be a good candidate to run interactively with Impala</li>
         </ul>
         </ul>
       </fieldset>
       </fieldset>
     </form>
     </form>

+ 26 - 13
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -331,12 +331,14 @@ def upload_history(request):
 def upload_table_stats(request):
 def upload_table_stats(request):
   response = {'status': -1}
   response = {'status': -1}
 
 
-  db_tables = json.loads(request.POST.get('dbTables'), '[]')
+  db_tables = json.loads(request.POST.get('db_tables'), '[]')
   source_platform = request.POST.get('sourcePlatform', 'hive')
   source_platform = request.POST.get('sourcePlatform', 'hive')
   with_columns = json.loads(request.POST.get('with_columns', 'false'))
   with_columns = json.loads(request.POST.get('with_columns', 'false'))
+  with_ddl = json.loads(request.POST.get('with_ddl', 'false'))
 
 
   table_stats = []
   table_stats = []
   column_stats = []
   column_stats = []
+  table_ddls = []
 
 
   for db_table in db_tables:
   for db_table in db_tables:
     path = _get_table_name(db_table)
     path = _get_table_name(db_table)
@@ -345,30 +347,41 @@ def upload_table_stats(request):
       full_table_stats = json.loads(get_table_stats(request, database=path['database'], table=path['table']).content)
       full_table_stats = json.loads(get_table_stats(request, database=path['database'], table=path['table']).content)
       stats = dict((stat['data_type'], stat['comment']) for stat in full_table_stats['stats'])
       stats = dict((stat['data_type'], stat['comment']) for stat in full_table_stats['stats'])
 
 
-      table_stats.append((db_table, stats.get('numRows', -1)))
+      table_stats.append({
+        'table_name': db_table,
+        #'avg_row_len':  stats.get('numRows', -1),
+        'num_rows':  stats.get('numRows', -1)
+      })
 
 
       if with_columns:
       if with_columns:
         for col in full_table_stats['columns']:
         for col in full_table_stats['columns']:
           col_stats = json.loads(get_table_stats(request, database=path['database'], table=path['table'], column=col).content)['stats']
           col_stats = json.loads(get_table_stats(request, database=path['database'], table=path['table'], column=col).content)['stats']
           col_stats = dict([(key, val) for col_stat in col_stats for key, val in col_stat.iteritems()])
           col_stats = dict([(key, val) for col_stat in col_stats for key, val in col_stat.iteritems()])
 
 
-          column_stats.append(
-              (db_table, col, col_stats['data_type'],
-               int(col_stats.get('distinct_count')) if col_stats.get('distinct_count') != '' else -1,
-               int(col_stats['num_nulls']) if col_stats['num_nulls'] != '' else -1,
-               int(float(col_stats['avg_col_len'])) if col_stats['avg_col_len'] != '' else -1
-            )
-          )
+          column_stats.append({
+            'table_name': db_table,
+            'column_name': col,
+            'data_type': col_stats['data_type'],
+            "num_distinct": int(col_stats.get('distinct_count')) if col_stats.get('distinct_count') != '' else -1,
+            "num_nulls": int(col_stats['num_nulls']) if col_stats['num_nulls'] != '' else -1,
+            "avg_col_len": int(float(col_stats['avg_col_len'])) if col_stats['avg_col_len'] != '' else -1
+          })
+
+#       if with_ddl:
+#         table_ddl.append((original_query_id, execution_time, query_data['snippets'][0]['statement']))
+
     except Exception, e:
     except Exception, e:
       LOG.exception('Skipping upload of %s: %s' % (db_table, e))
       LOG.exception('Skipping upload of %s: %s' % (db_table, e))
 
 
   api = OptimizerApi()
   api = OptimizerApi()
 
 
   response['upload_table_stats'] = api.upload(data=table_stats, data_type='table_stats', source_platform=source_platform)
   response['upload_table_stats'] = api.upload(data=table_stats, data_type='table_stats', source_platform=source_platform)
-  if with_columns:
+  response['status'] = 0 if response['upload_table_stats']['status']['state'] == 'FINISHED' else -1
+  if column_stats:
     response['upload_cols_stats'] = api.upload(data=column_stats, data_type='cols_stats', source_platform=source_platform)
     response['upload_cols_stats'] = api.upload(data=column_stats, data_type='cols_stats', source_platform=source_platform)
-
-  response['status'] = 0
+    response['status'] = response['status'] if response['upload_cols_stats']['status']['state'] == 'FINISHED' else -1
+  if table_ddls:
+    response['upload_table_ddl'] = api.upload(data=table_ddls, data_type='queries', source_platform=source_platform)
 
 
   return JsonResponse(response)
   return JsonResponse(response)
 
 
@@ -392,6 +405,6 @@ def _get_table_name(path):
   if '.' in path:
   if '.' in path:
     database, table = path.split('.', 1)
     database, table = path.split('.', 1)
   else:
   else:
-    database, table = '', path
+    database, table = 'default', path
 
 
   return {'database': database, 'table': table}
   return {'database': database, 'table': table}

+ 29 - 88
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -97,9 +97,18 @@ class OptimizerApi(object):
 
 
   def upload(self, data, data_type='queries', source_platform='generic', workload_id=None):
   def upload(self, data, data_type='queries', source_platform='generic', workload_id=None):
     if data_type in ('table_stats', 'cols_stats'):
     if data_type in ('table_stats', 'cols_stats'):
-      data_suffix = '.log'
+      data_suffix = '.json'
+      if data_type == 'table_stats':
+        extra_parameters = {'fileType': 'TABLE_STATS'}
+      else:
+        extra_parameters = {'fileType': 'COLUMN_STATS'}
     else:
     else:
       data_suffix = '.csv'
       data_suffix = '.csv'
+      extra_parameters = {
+          'colDelim': ',',
+          'rowDelim': '\n',
+          'headerFields': OptimizerApi.UPLOAD[data_type]['headerFields']
+      }
 
 
     f_queries_path = NamedTemporaryFile(suffix=data_suffix)
     f_queries_path = NamedTemporaryFile(suffix=data_suffix)
     f_queries_path.close() # Reopened as real file below to work well with the command
     f_queries_path.close() # Reopened as real file below to work well with the command
@@ -108,24 +117,29 @@ class OptimizerApi(object):
       f_queries = open(f_queries_path.name, 'w+')
       f_queries = open(f_queries_path.name, 'w+')
 
 
       try:
       try:
-        content_generator = OptimizerDataAdapter(data, data_type=data_type)
-        queries_csv = export_csvxls.create_generator(content_generator, 'csv')
+        # Queries
+        if data_suffix == '.csv':
+          content_generator = OptimizerQueryDataAdapter(data)
+          queries_csv = export_csvxls.create_generator(content_generator, 'csv')
 
 
-        for row in queries_csv:
-          f_queries.write(row)
+          for row in queries_csv:
+            f_queries.write(row)
+        else:
+          # Table, column stats
+          f_queries.write(json.dumps(data))
 
 
       finally:
       finally:
         f_queries.close()
         f_queries.close()
 
 
-      response = self._api.call_api('upload', {
+      parameters = {
           'tenant' : self._product_name,
           'tenant' : self._product_name,
           'fileLocation': f_queries.name,
           'fileLocation': f_queries.name,
           'sourcePlatform': source_platform,
           'sourcePlatform': source_platform,
-          'colDelim': ',',
-          'rowDelim': '\n',
-          'headerFields': OptimizerApi.UPLOAD[data_type]['headerFields']
-      })
+      }
+      parameters.update(extra_parameters)
+      response = self._api.call_api('upload', parameters)
       status = json.loads(response)
       status = json.loads(response)
+
       status['count'] = len(data)
       status['count'] = len(data)
       return status
       return status
 
 
@@ -134,6 +148,7 @@ class OptimizerApi(object):
     finally:
     finally:
       os.remove(f_queries_path.name)
       os.remove(f_queries_path.name)
 
 
+
   def upload_status(self, workload_id):
   def upload_status(self, workload_id):
     return self._api.call_api('uploadStatus', {'tenant' : self._product_name, 'workloadId': workload_id}).json()
     return self._api.call_api('uploadStatus', {'tenant' : self._product_name, 'workloadId': workload_id}).json()
 
 
@@ -234,90 +249,16 @@ class OptimizerApi(object):
               "name": "SQL_FULLTEXT"
               "name": "SQL_FULLTEXT"
           }
           }
       ]
       ]
-    },
-    'table_stats': {
-        'headers': ['TABLE_NAME', 'NUM_ROWS'],
-        "colDelim": ",",
-        "rowDelim": "\\n",
-        "headerFields": [
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "TABLE_NAME"
-            },
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "NUM_ROWS"
-            }
-        ]
-    },
-    'cols_stats': {
-        'headers': ['table_name', 'column_name', 'data_type', 'num_distinct', 'num_nulls', 'avg_col_len'], # Lower case for some reason
-        "colDelim": ",",
-        "rowDelim": "\\n",
-        "headerFields": [
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "table_name"
-            },
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "column_name"
-            },
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "data_type"
-            },
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "num_distinct"
-            },
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "num_nulls"
-            },
-            {
-                "count": 0,
-                "coltype": "NONE",
-                "use": True,
-                "tag": "",
-                "name": "avg_col_len"
-            }
-        ]
     }
     }
   }
   }
 
 
 
 
-def OptimizerDataAdapter(data, data_type='queries'):
-  headers = OptimizerApi.UPLOAD[data_type]['headers']
+def OptimizerQueryDataAdapter(data):
+  headers = OptimizerApi.UPLOAD['queries']['headers']
 
 
-  if data_type in ('table_stats', 'cols_stats'):
+  if data and len(data[0]) == 3:
     rows = data
     rows = data
   else:
   else:
-    if data and len(data[0]) == 3:
-      rows = data
-    else:
-      rows = ([str(uuid.uuid4()), 0.0, q] for q in data)
+    rows = ([str(uuid.uuid4()), 0.0, q] for q in data)
 
 
   yield headers, rows
   yield headers, rows
-

+ 24 - 0
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -1465,6 +1465,30 @@ var EditorViewModel = (function() {
       });
       });
     };
     };
 
 
+    huePubSub.subscribe("editor.table.stats.upload", function (activeTables) {
+      self.loadTableStats(activeTables);
+    });
+
+    self.loadTableStats = function (activeTables) {
+      logGA('load_table_stats');
+
+      $.post("/metadata/api/optimizer/upload/table_stats", {
+    	db_tables: ko.mapping.toJSON(activeTables),
+        sourcePlatform: ko.mapping.toJSON(self.type()),
+        with_columns: ko.mapping.toJSON(true),
+        with_ddl: ko.mapping.toJSON(true)
+      }, function(data) {
+        if (data.status == 0) {
+          $(document).trigger("info", activeTables + " stats uploaded successfully.");
+          if (data.upload_table_ddl) {
+            self.watchUploadStatus(data.upload_table_ddl.status.workloadId);
+          }
+        } else {
+          $(document).trigger("error", data.message);
+        }
+      });
+    };
+
     self.watchUploadStatus = function (workloadId) {
     self.watchUploadStatus = function (workloadId) {
       $.post("/metadata/api/optimizer/upload/status", {
       $.post("/metadata/api/optimizer/upload/status", {
         workloadId: workloadId
         workloadId: workloadId

+ 19 - 7
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1064,7 +1064,7 @@ from notebook.conf import ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, ENABLE_
   </div>
   </div>
 
 
 
 
-  <div class="context-panel" data-bind="css: {'visible': isContextPanelVisible}">
+  <div class="context-panel" data-bind="css: {'visible': isContextPanelVisible}" style="${ 'height: 100%' if not is_embeddable else '' }">
     <ul class="nav nav-tabs">
     <ul class="nav nav-tabs">
       % if has_optimizer():
       % if has_optimizer():
       <li class="active"><a href="#assistantTab" data-toggle="tab">${_('Assistant')}</a></li>
       <li class="active"><a href="#assistantTab" data-toggle="tab">${_('Assistant')}</a></li>
@@ -1078,13 +1078,25 @@ from notebook.conf import ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, ENABLE_
     </ul>
     </ul>
 
 
     <div class="tab-content" style="border: none">
     <div class="tab-content" style="border: none">
-      <div class="tab-pane ${ 'active' if not has_optimizer() else '' }" id="sessionsTab">
-        <div class="row-fluid">
-          <div class="span12" data-bind="template: { name: 'notebook-session-config-template', data: $root }"></div>
+      % if has_optimizer():
+      <div class="tab-pane ${ 'active' if has_optimizer() else '' }" id="assistantTab">
+        <div class="span12">
+          <form class="form-horizontal">
+            <fieldset>
+              <div data-bind="component: { name: 'assistant-panel' }"></div>
+            </fieldset>
+          </form>
         </div>
         </div>
       </div>
       </div>
+      % endif
 
 
-      % if ENABLE_QUERY_SCHEDULING.get():
+    <div class="tab-pane ${ 'active' if not has_optimizer() else '' }" id="sessionsTab">
+      <div class="row-fluid">
+        <div class="span12" data-bind="template: { name: 'notebook-session-config-template', data: $root }"></div>
+      </div>
+    </div>
+
+    % if ENABLE_QUERY_SCHEDULING.get():
       <!-- ko if: $root.selectedNotebook() && $root.selectedNotebook().isBatchable() -->
       <!-- ko if: $root.selectedNotebook() && $root.selectedNotebook().isBatchable() -->
       <!-- ko with: $root.selectedNotebook() -->
       <!-- ko with: $root.selectedNotebook() -->
       <div class="tab-pane" id="scheduleTab">
       <div class="tab-pane" id="scheduleTab">
@@ -1099,14 +1111,14 @@ from notebook.conf import ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, ENABLE_
         <br>
         <br>
         <div id="schedulerEditor"></div>
         <div id="schedulerEditor"></div>
         <!-- /ko -->
         <!-- /ko -->
+
         <!-- ko ifnot: isSaved() -->
         <!-- ko ifnot: isSaved() -->
         ${ _('Query needs to be saved first.') }
         ${ _('Query needs to be saved first.') }
         <!-- /ko -->
         <!-- /ko -->
       </div>
       </div>
       <!-- /ko -->
       <!-- /ko -->
       <!-- /ko -->
       <!-- /ko -->
-      % endif
-    </div>
+    % endif
   </div>
   </div>
 </script>
 </script>