Ver código fonte

HUE-5491 [metadata] Upload stats for columns of tables

Romain Rigaux 9 anos atrás
pai
commit
ccfb831

+ 1 - 0
apps/beeswax/src/beeswax/api.py

@@ -739,6 +739,7 @@ def get_table_stats(request, database, table, column=None):
   else:
     table = db.get_table(database, table)
     stats = table.stats
+    response['columns'] = [column.name for column in table.cols]
 
   response['stats'] = stats
   response['status'] = 0

+ 26 - 7
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -322,22 +322,41 @@ def upload_table_stats(request):
 
   db_tables = json.loads(request.POST.get('dbTables'), '[]')
   source_platform = request.POST.get('sourcePlatform', 'hive')
+  with_columns = json.loads(request.POST.get('with_columns', 'false'))
+
+  table_stats = []
+  column_stats = []
 
-  data = []
   for db_table in db_tables:
     path = _get_table_name(db_table)
 
     try:
-      table = get_table_stats(request, database=path['database'], table=path['table'])
-      stats = dict((stat['data_type'], stat['comment']) for stat in json.loads(table.content)['stats'])
-
-      data.append((db_table, stats.get('numRows', -1)))
+      full_table_stats = json.loads(get_table_stats(request, database=path['database'], table=path['table']).content)
+      stats = dict((stat['data_type'], stat['comment']) for stat in full_table_stats['stats'])
+
+      table_stats.append((db_table, stats.get('numRows', -1)))
+
+      if with_columns:
+        for col in full_table_stats['columns']:
+          col_stats = json.loads(get_table_stats(request, database=path['database'], table=path['table'], column=col).content)['stats']
+          col_stats = dict([(key, val) for col_stat in col_stats for key, val in col_stat.iteritems()])
+
+          column_stats.append(
+              (db_table, col, col_stats['data_type'],
+               int(col_stats.get('distinct_count')) if col_stats.get('distinct_count') != '' else -1,
+               int(col_stats['num_nulls']) if col_stats['num_nulls'] != '' else -1,
+               int(float(col_stats['avg_col_len'])) if col_stats['avg_col_len'] != '' else -1
+            )
+          )
     except Exception, e:
-      LOG.warning('Skipping upload of %s: %s' % (db_table, e))
+      LOG.exception('Skipping upload of %s: %s' % (db_table, e))
 
   api = OptimizerApi()
 
-  response['upload_history'] = api.upload(data=data, data_type='table_stats', source_platform=source_platform)
+  response['upload_table_stats'] = api.upload(data=table_stats, data_type='table_stats', source_platform=source_platform)
+  if with_columns:
+    response['upload_cols_stats'] = api.upload(data=column_stats, data_type='cols_stats', source_platform=source_platform)
+
   response['status'] = 0
 
   return JsonResponse(response)

+ 57 - 2
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -111,6 +111,61 @@ class OptimizerApi(object):
             "name": "NUM_ROWS"
         }
     ]
+}"""
+    },
+    'cols_stats': {
+        'headers': ['table_name', 'column_name', 'data_type', 'num_distinct', 'num_nulls', 'avg_col_len'], # Lower case for some reason
+        'file_headers': """{
+    "fileLocation": "%(query_file)s",
+    "tenant": "%(tenant)s",
+    "fileName": "%(query_file_name)s",
+    "sourcePlatform": "%(source_platform)s",
+    "colDelim": ",",
+    "rowDelim": "\\n",
+    "headerFields": [
+        {
+            "count": 0,
+            "coltype": "NONE",
+            "use": true,
+            "tag": "",
+            "name": "table_name"
+        },
+        {
+            "count": 0,
+            "coltype": "NONE",
+            "use": true,
+            "tag": "",
+            "name": "column_name"
+        },
+        {
+            "count": 0,
+            "coltype": "NONE",
+            "use": true,
+            "tag": "",
+            "name": "data_type"
+        },
+        {
+            "count": 0,
+            "coltype": "NONE",
+            "use": true,
+            "tag": "",
+            "name": "num_distinct"
+        },
+        {
+            "count": 0,
+            "coltype": "NONE",
+            "use": true,
+            "tag": "",
+            "name": "num_nulls"
+        },
+        {
+            "count": 0,
+            "coltype": "NONE",
+            "use": true,
+            "tag": "",
+            "name": "avg_col_len"
+        }
+    ]
 }"""
     }
   }
@@ -213,7 +268,7 @@ class OptimizerApi(object):
   def upload(self, data, data_type='queries', source_platform='generic', workload_id=None):
     data_headers = OptimizerApi.UPLOAD[data_type]['file_headers']
 
-    if data_type == 'table_stats':
+    if data_type in ('table_stats', 'cols_stats'):
       data_suffix = '.log'
     else:
       data_suffix = '.csv'
@@ -361,7 +416,7 @@ class OptimizerApi(object):
 def OptimizerDataAdapter(data, data_type='queries'):
   headers = OptimizerApi.UPLOAD[data_type]['headers']
 
-  if data_type == 'table_stats':
+  if data_type in ('table_stats', 'cols_stats'):
     rows = data
   else:
     if data and len(data[0]) == 3: