Pārlūkot izejas kodu

HUE-6968 [metadata] Gather column stats via Impala

Romain Rigaux 8 gadi atpakaļ
vecāks
revīzija
7b15b90

+ 14 - 9
apps/beeswax/src/beeswax/server/dbms.py

@@ -461,15 +461,11 @@ class HiveServer2Dbms(object):
       data = list(result.rows())
 
       if self.server_name == 'impala':
-        data = [col for col in data if col[0] == column][0]
-        return [
-            {'col_name': data[0]},
-            {'data_type': data[1]},
-            {'distinct_count': data[2]},
-            {'num_nulls': data[3]},
-            {'max_col_len': data[4]},
-            {'avg_col_len': data[5]},
-        ]
+        if column == -1: # All the columns
+          return [self._extract_impala_column(col) for col in data]
+        else:
+          data = [col for col in data if col[0] == column][0]
+          return self._extract_impala_column(data)
       else:
         return [
             {'col_name': data[2][0]},
@@ -486,6 +482,15 @@ class HiveServer2Dbms(object):
     else:
       return []
 
+  def _extract_impala_column(self, col):
+    return [
+        {'col_name': col[0]},
+        {'data_type': col[1]},
+        {'distinct_count': col[2]},
+        {'num_nulls': col[3]},
+        {'max_col_len': col[4]},
+        {'avg_col_len': col[5]},
+    ]
 
   def get_table_properties(self, database, table, property_name=None):
     hql = 'SHOW TBLPROPERTIES `%s`.`%s`' % (database, table)

+ 11 - 4
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -424,13 +424,20 @@ def upload_table_stats(request):
       })
 
       if with_columns:
-        for col in full_table_stats['columns'][:25]:
-          col_stats = json.loads(get_table_stats(request, database=path['database'], table=path['table'], column=col).content)['stats']
-          col_stats = dict([(key, val) for col_stat in col_stats for key, val in col_stat.iteritems()])
+        if source_platform == 'impala':
+          colum_stats = json.loads(get_table_stats(mock_request, database=path['database'], table=path['table'], column=-1).content)['stats']
+        else:
+          colum_stats = [
+              json.loads(get_table_stats(mock_request, database=path['database'], table=path['table'], column=col).content)['stats']
+              for col in full_table_stats['columns'][:25]
+          ]
 
+        raw_column_stats = [dict([(key, val if val is not None else '') for col_stat in col for key, val in col_stat.iteritems()]) for col in colum_stats]
+
+        for col_stats in raw_column_stats:
           column_stats.append({
             'table_name': '%(database)s.%(table)s' % path, # DB Prefix
-            'column_name': col,
+            'column_name': col_stats['col_name'],
             'data_type': col_stats['data_type'],
             "num_distinct": int(col_stats.get('distinct_count')) if col_stats.get('distinct_count') != '' else -1,
             "num_nulls": int(col_stats['num_nulls']) if col_stats['num_nulls'] != '' else -1,