浏览代码

HUE-1084 [beeswax] Integrate table and column stats

Romain Rigaux 10 年之前
父节点
当前提交
d336e198fe

+ 0 - 1
apps/beeswax/src/beeswax/api.py

@@ -15,7 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import logging
 import re
 

+ 23 - 8
apps/beeswax/src/beeswax/server/dbms.py

@@ -199,16 +199,31 @@ class HiveServer2Dbms(object):
         return result
 
 
-  def analyze_table_table(self, database, table):
-    hql = 'analyze table `%(database)s.%(table_name)` compute statistics' % {'database': database, 'table_name': table.name}
-    query = hql_query(hql, database)
+  def analyze_table(self, database, table):
+    hql = 'ANALYZE TABLE `%(database)s`.`%(table)s` COMPUTE STATISTICS' % {'database': database, 'table': table}
+
+    return self.execute_statement(hql)
 
-    return self.execute_query(query)
+
+  def analyze_table_columns(self, database, table):
+    hql = 'ANALYZE TABLE `%(database)s`.`%(table)s` COMPUTE STATISTICS FOR COLUMNS' % {'database': database, 'table': table}
+
+    return self.execute_statement(hql)
 
 
-  def analyze_table_column(self):
-    # analyze table <table_name> partition <part_name> compute statistics for columns <col_name1>, <col_name2>...
-    pass
+  def get_table_columns_stats(self, database, table, column):
+    hql = 'DESCRIBE FORMATTED `%(database)s`.`%(table)s` %(column)s' % {'database': database, 'table': table, 'column': column}
+
+    query = hql_query(hql)
+    handle = self.execute_and_wait(query, timeout_sec=5.0)
+
+    if handle:
+      result = self.fetch(handle, rows=100)
+      self.close(handle)
+      return [col for table in result.rows() for col in table]
+    else:
+      return []
+
 
   def drop_table(self, database, table):
     if table.is_view:
@@ -263,7 +278,7 @@ class HiveServer2Dbms(object):
       try:
         hql = "INVALIDATE METADATA %s.%s" % (database, table,)
         query = hql_query(hql, database, query_type=QUERY_TYPES[1])
-  
+
         handle = self.execute_and_wait(query, timeout_sec=10.0)
       except Exception, e:
         LOG.warn('Refresh tables cache out of sync: %s' % smart_str(e))

+ 13 - 1
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -124,7 +124,19 @@ class HiveServerTable(Table):
     rows = self.describe
     col_row_index = 2
     end_cols_index = map(itemgetter('col_name'), rows[col_row_index:]).index('')
-    return rows[col_row_index + end_cols_index + 1:]
+    return [{
+          'col_name': prop['col_name'].strip() if prop['col_name'] else prop['col_name'],
+          'data_type': prop['data_type'].strip() if prop['data_type'] else prop['data_type'],
+          'comment': prop['comment'].strip() if prop['comment'] else prop['comment']
+        } for prop in rows[col_row_index + end_cols_index + 1:]
+    ]
+
+  @property
+  def stats(self):
+    rows = self.properties
+    col_row_index = map(itemgetter('col_name'), rows).index('Table Parameters:') + 1
+    end_cols_index = map(itemgetter('data_type'), rows[col_row_index:]).index(None)
+    return rows[col_row_index:][:end_cols_index]
 
 
 class HiveServerTRowSet2:

+ 21 - 3
apps/beeswax/src/beeswax/templates/execute.mako

@@ -1177,9 +1177,16 @@ $(document).ready(function () {
             var _table = $("<li>");
             var _metastoreLink = "";
             % if has_metastore:
-              _metastoreLink = "<i class='fa fa-eye' title='" + "${ _('View in Metastore Browser') }" + "'></i>";
+              _metastoreLink = "<i class='fa fa-bar-chart' title='" + "${ _('View statistics') }" + "'></i>";
             % endif
-            _table.html("<a href='javascript:void(0)' class='pull-right' style='padding-right:5px'><i class='fa fa-list' title='" + "${ _('Preview Sample data') }" + "' style='margin-left:5px'></i></a><a href='/metastore/table/" + viewModel.database() + "/" + table + "' target='_blank' class='pull-right hide'>" + _metastoreLink + "</a><div><a href='javascript:void(0)' title='" + table + "'><i class='fa fa-table'></i> " + table + "</a><ul class='unstyled'></ul></div>");
+            _table.html("<a href='javascript:void(0)' class='pull-right' style='padding-right:5px'><i class='fa fa-list' title='" + "${ _('Preview Sample data') }" + "' style='margin-left:5px'></i></a>" +
+            "<a id='stats-analysis' href='javascript:void(0)' class='pull-right'>" + _metastoreLink + "</a>" +
+
+          "<div id='stats-analysis-content' class='hide'>" +
+            "todo" +
+          "</div>" +
+
+            "<div><a href='javascript:void(0)' title='" + table + "'><i class='fa fa-table'></i> " + table + "</a><ul class='unstyled'></ul></div>");
 
             _table.data("table", table).attr("id", "navigatorTables_" + table);
             _table.find("a:eq(2)").on("click", function () {
@@ -2355,7 +2362,6 @@ $(document).ready(function () {
     });
   });
 
-  // Help.
   $("#help").popover({
     'title': "${_('Did you know?')}",
     'content': $("#help-content").html(),
@@ -2364,6 +2370,14 @@ $(document).ready(function () {
     'html': true
   });
 
+  $("#stats-analysis").popover({
+    'title': "${_('Did you know?')}",
+    'content': $("#stats-analysis-content").html(),
+    'trigger': 'click',
+    'placement': 'left',
+    'html': true
+  });
+
   $("#hdfs-directory-help").popover({
     'title': "${_('Did you know?')}",
     'content': $("#hdfs-directory-help-content").html(),
@@ -2605,6 +2619,10 @@ $(document).ready(function () {
       routie('query');
     }
   });
+
+  $('#stats-analysis').click(function() {
+// todo
+  });
 });
 
 

+ 31 - 0
apps/metastore/src/metastore/tests.py

@@ -57,6 +57,7 @@ def _make_query(client, query, submission_type="Execute",
 
   return res
 
+
 class TestMetastoreWithHadoop(BeeswaxSampleProvider):
   requires_hadoop = True
 
@@ -248,3 +249,33 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
 
     check(client, [200, 302]) # Ok
+
+
+  def test_analyze_table_and_read_statistics(self):
+    # No stats
+    resp = self.client.get(reverse('metastore:get_table_stats', kwargs={'database': 'default', 'table': 'test'}))
+    stats = json.loads(resp.content)['stats']
+    assert_equal('COLUMN_STATS_ACCURATE', stats[0]['data_type'], resp.content)
+
+    resp = self.client.get(reverse('metastore:get_table_stats', kwargs={'database': 'default', 'table': 'test', 'column': 'foo'}))
+    stats = json.loads(resp.content)['stats']
+    assert_equal(["foo", "int", "", "", "", "", "", "", "", "", "from deserializer"], stats[-11:])
+
+    # Compute stats
+    response = self.client.post(reverse("metastore:analyze_table", kwargs={'database': 'default', 'table': 'test'}), follow=True)
+    response = wait_for_query_to_finish(self.client, response, max=60.0)
+    assert_true(response, response)
+
+    response = self.client.post(reverse("metastore:analyze_table", kwargs={'database': 'default', 'table': 'test', 'columns': True}), follow=True)
+    response = wait_for_query_to_finish(self.client, response, max=60.0)
+    assert_true(response, response)
+
+    # Retrieve stats
+    resp = self.client.get(reverse('metastore:get_table_stats', kwargs={'database': 'default', 'table': 'test'}))
+    stats = json.loads(resp.content)['stats']
+    assert_true(any([stat for stat in stats if stat['data_type'] == 'numRows']), resp.content)
+    assert_true(any([stat for stat in stats if stat['comment'] == '256']), resp.content)
+
+    resp = self.client.get(reverse('metastore:get_table_stats', kwargs={'database': 'default', 'table': 'test', 'column': 'foo'}))
+    stats = json.loads(resp.content)['stats']
+    assert_equal(["foo", "int", "0", "255", "0", "180", "", "", "", "", "from deserializer"], stats[-11:])

+ 4 - 1
apps/metastore/src/metastore/urls.py

@@ -30,5 +30,8 @@ urlpatterns = patterns('metastore.views',
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/load$', 'load_table', name='load_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/read$', 'read_table', name='read_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_id>\w+)$', 'read_partition', name='read_partition'),
-  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)?$', 'analyze_table', name='analyze_table'),
+
+  # API
+  url(r'^analyze/(?P<database>\w+)/(?P<table>\w+)/(?P<columns>\w+)?$', 'analyze_table', name='analyze_table'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/stats/(?P<column>\w+)?$', 'get_table_stats', name='get_table_stats'),
 )

+ 33 - 17
apps/metastore/src/metastore/views.py

@@ -271,42 +271,58 @@ def describe_partitions(request, database, table):
 
   partitions = db.get_partitions(database, table_obj, max_parts=None)
 
-  return render("describe_partitions.mako", request,
-      {'breadcrumbs': [
-        {
+  return render("describe_partitions.mako", request, {
+    'breadcrumbs': [{
           'name': database,
           'url': reverse('metastore:show_tables', kwargs={'database': database})
-        },
-        {
+        }, {
           'name': table,
           'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
-        },
-        {
+        },{
           'name': 'partitions',
           'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
         },
       ],
-      'database': database, 'table': table_obj, 'partitions': partitions, 'request': request})
+      'database': database, 'table': table_obj, 'partitions': partitions, 'request': request
+  })
 
 
-def analyze_table(request, database, table, column=None):
+def analyze_table(request, database, table, columns=None):
   app_name = get_app_name(request)
   query_server = get_query_server_config(app_name)
   db = dbms.get(request.user, query_server)
 
   response = {'status': -1, 'message': '', 'redirect': ''}
 
-  if request.POST:
-    if column is None:
+  if request.method == "POST":
+    if columns is not None:
       query_history = db.analyze_table(database, table)
-      response['redirect'] = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + \
-                                     '?on_success_url=' + reverse('metastore:describe_table',
-                                                                  kwargs={'database': database, 'table': table.name})
-      response['status'] = 0
     else:
-      response['message'] = _('Column analysis not supportet yet')
+      query_history = db.analyze_table_columns(database, table)
+
+    response['watch_url'] = reverse('beeswax:api_watch_query_refresh_json', kwargs={'id': query_history.id})
+    response['status'] = 0
   else:
-    response['message'] = _('A POST request is required')
+    response['message'] = _('A POST request is required.')
+
+  return JsonResponse(response)
+
+
+def get_table_stats(request, database, table, column=None):
+  app_name = get_app_name(request)
+  query_server = get_query_server_config(app_name)
+  db = dbms.get(request.user, query_server)
+
+  response = {'status': -1, 'message': '', 'redirect': ''}
+
+  if column is not None:
+    stats = db.get_table_columns_stats(database, table, column)
+  else:
+    table = db.get_table(database, table)
+    stats = table.stats
+
+  response['stats'] = stats
+  response['status'] = 0
 
   return JsonResponse(response)