瀏覽代碼

HUE-1084 [metastore] Add a top column value API

Slow with Hive
Romain Rigaux 10 年之前
父節點
當前提交
fae2534

+ 21 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -225,6 +225,27 @@ class HiveServer2Dbms(object):
       return []
 
 
+  def get_top_terms(self, database, table, column, limit=30, prefix=None):
+    limit = max(limit, 100)
+    prefix_match = ''
+    if prefix:
+      prefix_match = "WHERE %(column)s LIKE '%(prefix)s%%'" % {'column': column, 'prefix': prefix}
+
+    hql = 'SELECT %(column)s, COUNT(*) AS ct FROM `%(database)s`.`%(table)s` %(prefix_match)s GROUP BY %(column)s ORDER BY ct DESC LIMIT %(limit)s' % {
+        'database': database, 'table': table, 'column': column, 'prefix_match': prefix_match, 'limit': limit,
+    }
+
+    query = hql_query(hql)
+    handle = self.execute_and_wait(query, timeout_sec=60.0) # Hive is very slow
+
+    if handle:
+      result = self.fetch(handle, rows=limit)
+      self.close(handle)
+      return list(result.rows())
+    else:
+      return []
+
+
   def drop_table(self, database, table):
     if table.is_view:
       hql = "DROP VIEW `%s.%s`" % (database, table.name,)

+ 23 - 21
apps/beeswax/src/beeswax/tests.py

@@ -1956,36 +1956,38 @@ class TestHiveServer2API():
 
     assert_equal([
         {'comment': None, 'col_name': '# Partition Information', 'data_type': None},
-        {'comment': 'comment             ', 'col_name': '# col_name            ', 'data_type': 'data_type           '},
+        {'comment': 'comment', 'col_name': '# col_name', 'data_type': 'data_type'},
         {'comment': None, 'col_name': '', 'data_type': None},
         {'comment': '', 'col_name': 'baz', 'data_type': 'string'},
         {'comment': '', 'col_name': 'boom', 'data_type': 'string'},
         {'comment': None, 'col_name': '', 'data_type': None},
         {'comment': None, 'col_name': '# Detailed Table Information', 'data_type': None},
-        {'comment': None, 'col_name': 'Database:           ', 'data_type': 'default             '},
-        {'comment': None, 'col_name': 'Owner:              ', 'data_type': 'romain              '},
-        {'comment': None, 'col_name': 'CreateTime:         ', 'data_type': 'Wed Aug 13 13:39:53 PDT 2014'},
-        {'comment': None, 'col_name': 'LastAccessTime:     ', 'data_type': 'UNKNOWN             '},
-        {'comment': None, 'col_name': 'Protect Mode:       ', 'data_type': 'None                '},
-        {'comment': None, 'col_name': 'Retention:          ', 'data_type': '0                   '},
-        {'comment': None, 'col_name': 'Location:           ', 'data_type': 'hdfs://localhost:8020/user/hive/warehouse/test_partitions'},
-        {'comment': None, 'col_name': 'Table Type:         ', 'data_type': 'MANAGED_TABLE       '},
+        {'comment': None, 'col_name': 'Database:', 'data_type': 'default'},
+        {'comment': None, 'col_name': 'Owner:', 'data_type': 'romain'},
+        {'comment': None, 'col_name': 'CreateTime:', 'data_type': 'Wed Aug 13 13:39:53 PDT 2014'},
+        {'comment': None, 'col_name': 'LastAccessTime:', 'data_type': 'UNKNOWN'},
+        {'comment': None, 'col_name': 'Protect Mode:', 'data_type': 'None'},
+        {'comment': None, 'col_name': 'Retention:', 'data_type': '0'},
+        {'comment': None, 'col_name': 'Location:', 'data_type': 'hdfs://localhost:8020/user/hive/warehouse/test_partitions'},
+        {'comment': None, 'col_name': 'Table Type:', 'data_type': 'MANAGED_TABLE'},
         {'comment': None, 'col_name': 'Table Parameters:', 'data_type': None},
-        {'comment': '1407962393          ', 'col_name': '', 'data_type': 'transient_lastDdlTime'},
+        {'comment': '1407962393', 'col_name': '', 'data_type': 'transient_lastDdlTime'},
         {'comment': None, 'col_name': '', 'data_type': None},
         {'comment': None, 'col_name': '# Storage Information', 'data_type': None},
-        {'comment': None, 'col_name': 'SerDe Library:      ', 'data_type': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'},
-        {'comment': None, 'col_name': 'InputFormat:        ', 'data_type': 'org.apache.hadoop.mapred.TextInputFormat'},
-        {'comment': None, 'col_name': 'OutputFormat:       ', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'},
-        {'comment': None, 'col_name': 'Compressed:         ', 'data_type': 'No                  '},
-        {'comment': None, 'col_name': 'Num Buckets:        ', 'data_type': '-1                  '},
-        {'comment': None, 'col_name': 'Bucket Columns:     ', 'data_type': '[]                  '},
-        {'comment': None, 'col_name': 'Sort Columns:       ', 'data_type': '[]                  '},
+        {'comment': None, 'col_name': 'SerDe Library:', 'data_type': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'},
+        {'comment': None, 'col_name': 'InputFormat:', 'data_type': 'org.apache.hadoop.mapred.TextInputFormat'},
+        {'comment': None, 'col_name': 'OutputFormat:', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'},
+        {'comment': None, 'col_name': 'Compressed:', 'data_type': 'No'},
+        {'comment': None, 'col_name': 'Num Buckets:', 'data_type': '-1'},
+        {'comment': None, 'col_name': 'Bucket Columns:', 'data_type': '[]'},
+        {'comment': None, 'col_name': 'Sort Columns:', 'data_type': '[]'},
         {'comment': None, 'col_name': 'Storage Desc Params:', 'data_type': None},
-        {'comment': '\\t                  ', 'col_name': '', 'data_type': 'field.delim         '},
-        {'comment': '\\n                  ', 'col_name': '', 'data_type': 'line.delim          '},
-        {'comment': '\\t                  ', 'col_name': '', 'data_type': 'serialization.format'}],
-            table.properties)
+        {'comment': '\\t', 'col_name': '', 'data_type': 'field.delim'},
+        {'comment': '\\n', 'col_name': '', 'data_type': 'line.delim'},
+        {'comment': '\\t', 'col_name': '', 'data_type': 'serialization.format'}
+        ],
+        table.properties
+    )
 
     assert_equal('hdfs://localhost:8020/user/hive/warehouse/test_partitions', table.path_location)
 

+ 10 - 0
apps/metastore/src/metastore/tests.py

@@ -279,3 +279,13 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     resp = self.client.get(reverse('metastore:get_table_stats', kwargs={'database': 'default', 'table': 'test', 'column': 'foo'}))
     stats = json.loads(resp.content)['stats']
     assert_equal(["foo", "int", "0", "255", "0", "180", "", "", "", "", "from deserializer"], stats[2])
+
+
+  def test_get_top_terms(self):
+    resp = self.client.get(reverse("metastore:get_top_terms", kwargs={'database': 'default', 'table': 'test', 'column': 'foo'}))
+    terms = json.loads(resp.content)['terms']
+    assert_equal([[255, 1], [254, 1], [253, 1], [252, 1]], terms[:4])
+
+    resp = self.client.get(reverse("metastore:get_top_terms", kwargs={'database': 'default', 'table': 'test', 'column': 'foo', 'prefix': '10'}))
+    terms = json.loads(resp.content)['terms']
+    assert_equal([[109, 1], [108, 1], [107, 1], [106, 1]], terms[:4])

+ 1 - 0
apps/metastore/src/metastore/urls.py

@@ -34,4 +34,5 @@ urlpatterns = patterns('metastore.views',
   # API
   url(r'^analyze/(?P<database>\w+)/(?P<table>\w+)/(?P<columns>\w+)?$', 'analyze_table', name='analyze_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/stats/(?P<column>\w+)?$', 'get_table_stats', name='get_table_stats'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/terms/(?P<column>\w+)/(?P<prefix>\w+)?$', 'get_top_terms', name='get_top_terms'),
 )

+ 20 - 5
apps/metastore/src/metastore/views.py

@@ -252,11 +252,11 @@ def load_table(request, database, table):
 
   if response['status'] == -1:
     popup = render('popups/load_data.mako', request, {
-                     'table': table,
-                     'load_form': load_form,
-                     'database': database,
-                     'app_name': 'beeswax'
-                 }, force_template=True).content
+           'table': table,
+           'load_form': load_form,
+           'database': database,
+           'app_name': 'beeswax'
+       }, force_template=True).content
     response['data'] = popup
 
   return JsonResponse(response)
@@ -327,5 +327,20 @@ def get_table_stats(request, database, table, column=None):
   return JsonResponse(response)
 
 
+def get_top_terms(request, database, table, column, prefix=None):
+  app_name = get_app_name(request)
+  query_server = get_query_server_config(app_name)
+  db = dbms.get(request.user, query_server)
+
+  response = {'status': -1, 'message': '', 'redirect': ''}
+
+  terms = db.get_top_terms(database, table, column, prefix=prefix)
+
+  response['terms'] = terms
+  response['status'] = 0
+
+  return JsonResponse(response)
+
+
 def has_write_access(user):
   return user.is_superuser or user.has_hue_permission(action="write", app=DJANGO_APPS[0])