Browse Source

HUE-1275 [metastore] Show table details

Beginning of HUE-1084 [beeswax] Integrate table and column stats
Romain Rigaux 12 years ago
parent
commit
7953b1b0e9

+ 10 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -129,6 +129,7 @@ class Dbms:
     if no_start_over_support:
       start_over = False
 
+
     return self.client.fetch(query_handle, start_over, rows)
 
 
@@ -159,6 +160,15 @@ class Dbms:
       if handle:
         return self.fetch(handle)
 
+  def analyze_table_table(self, database, table):
+    hql = 'analyze table `%(database)s.%(table_name)` compute statistics' % {'database': database, 'table_name': table.name}
+    query = hql_query(hql, database)
+
+    return self.execute_query(query)
+
+  def analyze_table_column(self):
+    # analyze table <table_name> partition <part_name> compute statistics for columns <col_name1>, <col_name2>...
+    pass
 
   def drop_table(self, database, table):
     if table.is_view:

+ 17 - 2
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -21,7 +21,6 @@ import thrift
 
 from operator import itemgetter
 
-from desktop.conf import KERBEROS
 from desktop.lib import thrift_util
 from hadoop import cluster
 
@@ -112,10 +111,26 @@ class HiveServerTable(Table):
     describe_text = rows[detailed_row_index]['data_type']
     try:
       # LazySimpleSerDe case
-      return describe_text+  rows[detailed_row_index + 1]['col_name']
+      return describe_text + rows[detailed_row_index + 1]['col_name']
     except:
       return describe_text
 
+  @property
+  def properties(self):
+    # Ugly but would need a recursive parsing to be clean
+    no_table = re.sub('\)$', '', re.sub('^Table\(', '', self.extended_describe))
+    properties = re.sub(', sd:StorageDescriptor\(cols.+?\]', '', no_table).split(', ')
+    props = []
+
+    for prop in properties:
+      key_val = prop.rsplit(':', 1)
+      if len(key_val) == 1:
+        key_val = key_val[0].rsplit('=', 1)
+      if len(key_val) == 2:
+        props.append(key_val)
+
+    return props
+
 
 class HiveServerTRowSet:
   def __init__(self, row_set, schema):

+ 49 - 2
apps/beeswax/src/beeswax/tests.py

@@ -58,7 +58,7 @@ from beeswax.models import SavedQuery, QueryHistory, HQL
 from beeswax.server import dbms
 from beeswax.server.dbms import QueryServerException
 from beeswax.server.hive_server2_lib import HiveServerClient,\
-  PartitionValueCompatible
+  PartitionValueCompatible, HiveServerTable
 from beeswax.test_base import BeeswaxSampleProvider
 
 
@@ -1426,7 +1426,7 @@ def test_split_statements():
   assert_equal(['select', "select * where id == '\"10;\"\"\"' limit 100"], hql_query("select; select * where id == '\"10;\"\"\"' limit 100;").statements)
 
 
-class MockHiveServerTable():
+class MockHiveServerTable(HiveServerTable):
 
   def __init__(self, data):
     self.path_location = data.get('path_location')
@@ -1440,6 +1440,53 @@ class TestHiveServer2API():
     assert_equal(['2013022516'], PartitionValueCompatible(['datehour=2013022516'], table).values)
     assert_equal(['2011-07', '2011-07-01', '12'], PartitionValueCompatible(['month=2011-07/dt=2011-07-01/hr=12'], table).values)
 
+  def test_table_properties(self):
+    table = MockHiveServerTable({})
+    prev_extended_describe = getattr(MockHiveServerTable, 'extended_describe')
+
+    try:
+      extended_describe = 'Table(tableName:page_view, dbName:default, owner:romain, createTime:1360732885, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:viewtime, type:int, comment:null), FieldSchema(name:userid, type:bigint, comment:null), FieldSchema(name:page_url, type:string, comment:null), FieldSchema(name:referrer_url, type:string, comment:null), FieldSchema(name:ip, type:string, comment:IP Address of the User), FieldSchema(name:dt, type:string, comment:null), FieldSchema(name:country, type:string, comment:null)], location:hdfs://localhost:8020/user/hive/warehouse/page_view, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}, skewedInfo:SkewedInfo(skewedColNames:[], skewedColValues:[], skewedColValueLocationMaps:{}), storedAsSubDirectories:false), partitionKeys:[FieldSchema(name:dt, type:string, comment:null), FieldSchema(name:country, type:string, comment:null)], parameters:{numPartitions=0, numFiles=1, transient_lastDdlTime=1360732885, comment=This is the page view table}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE)'
+      setattr(table, 'extended_describe', extended_describe)
+
+      assert_equal([['tableName', 'page_view'],
+                    ['dbName', 'default'],
+                    ['owner', 'romain'],
+                    ['createTime', '1360732885'],
+                    ['lastAccessTime', '0'],
+                    ['retention', '0'],
+                    ['location:hdfs://localhost', '8020/user/hive/warehouse/page_view'],
+                    ['inputFormat', 'org.apache.hadoop.mapred.TextInputFormat'],
+                    ['outputFormat', 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'],
+                    ['compressed', 'false'],
+                    ['numBuckets', '-1'],
+                    ['serdeInfo:SerDeInfo(name', 'null'],
+                    ['serializationLib', 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'],
+                    ['parameters', '{serialization.format=1})'],
+                    ['bucketCols', '[]'],
+                    ['sortCols', '[]'],
+                    ['parameters', '{}'],
+                    ['skewedInfo:SkewedInfo(skewedColNames', '[]'],
+                    ['skewedColValues', '[]'],
+                    ['skewedColValueLocationMaps', '{})'],
+                    ['storedAsSubDirectories', 'false)'],
+                    ['partitionKeys:[FieldSchema(name', 'dt'],
+                    ['type', 'string'],
+                    ['comment', 'null)'],
+                    ['FieldSchema(name', 'country'],
+                    ['type', 'string'],
+                    ['comment', 'null)]'],
+                    ['parameters', '{numPartitions=0'],
+                    ['numFiles', '1'],
+                    ['transient_lastDdlTime', '1360732885'],
+                    ['comment', 'This is the page view table}'],
+                    ['viewOriginalText', 'null'],
+                    ['viewExpandedText', 'null'],
+                    ['tableType', 'MANAGED_TABLE']
+                  ],
+                  table.properties)
+    finally:
+      setattr(table, 'extended_describe', prev_extended_describe)
+
 
 class MockDbms:
 

+ 19 - 0
apps/metastore/src/metastore/templates/describe_table.mako

@@ -88,6 +88,7 @@ ${ components.menubar() }
               % if sample is not None:
               <li><a href="#sample" data-toggle="tab">${_('Sample')}</a></li>
               % endif
+              <li><a href="#properties" data-toggle="tab">${ _('Properties') }</a></li>
             </ul>
 
             <div class="tab-content">
@@ -136,6 +137,24 @@ ${ components.menubar() }
               % endif
               </div>
               % endif
+
+              <div class="tab-pane" id="properties">
+                <table class="table table-striped table-condensed">
+                  <thead>
+                    <tr>
+                      <th>${ _('Name') }</th>
+                      <th>${ _('Value') }</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                    % for name, value in table.properties:
+                      <tr>
+                        <td>${ name }</td>
+                        <td>${ value }</td>
+                      </tr>
+                     % endfor
+                  </tbody>
+              </div>
             </div>
           </p>
         </div>

+ 1 - 0
apps/metastore/src/metastore/tests.py

@@ -85,6 +85,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     # And have detail
     response = self.client.get("/metastore/table/default/test")
     assert_true("foo" in response.content)
+    assert_true("serdeInfo:SerDeInfo" in response.content, response.content)
 
     # Remember the number of history items. Use a generic fragment 'test' to pass verification.
     history_cnt = verify_history(self.client, fragment='test')

+ 1 - 0
apps/metastore/src/metastore/urls.py

@@ -30,4 +30,5 @@ urlpatterns = patterns('metastore.views',
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/load$', 'load_table', name='load_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/read$', 'read_table', name='read_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_id>\w+)$', 'read_partition', name='read_partition'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)?$', 'analyze_table', name='analyze_table'),
 )

+ 23 - 4
apps/metastore/src/metastore/views.py

@@ -15,10 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-  import json
-except ImportError:
-  import simplejson as json
+import json
 import logging
 
 from django.http import HttpResponse
@@ -279,5 +276,27 @@ def describe_partitions(request, database, table):
       'database': database, 'table': table_obj, 'partitions': partitions, 'request': request})
 
 
+def analyze_table(request, database, table, column=None):
+  app_name = get_app_name(request)
+  query_server = get_query_server_config(app_name)
+  db = dbms.get(request.user, query_server)
+
+  response = {'status': -1, 'message': '', 'redirect': ''}
+
+  if request.POST:
+    if column is None:
+      query_history = db.analyze_table(database, table)
+      response['redirect'] = reverse('beeswax:watch_query', args=[query_history.id]) + \
+                                     '?on_success_url=' + reverse('metastore:describe_table',
+                                                                  kwargs={'database': database, 'table': table.name})
+      response['status'] = 0
+    else:
+      response['message'] = _('Column analysis not supportet yet')
+  else:
+    response['message'] = _('A POST request is required')
+
+  return HttpResponse(json.dumps(response), mimetype="application/json")
+
+
 def has_write_access(user):
   return user.is_superuser or not user.has_hue_permission(action="read_only_access", app=DJANGO_APPS[0])