Browse Source

HUE-2554 [metastore] Use DESCRIBE FORMATTED to get table metadata

Big cleanup
Add some tests
Add DB prefix to SHOW PARTITIONS
Remove a DeprecationWarning: BaseException.message has been deprecated as of Python 2.6 exception.__class__, exception.message
Romain Rigaux 10 years ago
parent
commit
cba1ebc1d6

+ 2 - 5
apps/beeswax/src/beeswax/server/dbms.py

@@ -118,9 +118,6 @@ class HiveServer2Dbms(object):
     self.server_name = self.client.query_server['server_name']
     self.server_name = self.client.query_server['server_name']
 
 
   def get_table(self, database, table_name):
   def get_table(self, database, table_name):
-    # DB name not supported in SHOW PARTITIONS required in Table
-    self.use(database)
-
     return self.client.get_table(database, table_name)
     return self.client.get_table(database, table_name)
 
 
 
 
@@ -553,8 +550,8 @@ def expand_exception(exc, db, handle=None):
     # Always show something, even if server has died on the job.
     # Always show something, even if server has died on the job.
     log = _("Could not retrieve logs: %s." % e)
     log = _("Could not retrieve logs: %s." % e)
 
 
-  if not hasattr(exc, 'message') or not exc.message:
+  if not exc.args or not exc.args[0]:
     error_message = _("Unknown exception.")
     error_message = _("Unknown exception.")
   else:
   else:
-    error_message = force_unicode(exc.message, strings_only=True, errors='replace')
+    error_message = force_unicode(exc.args[0], strings_only=True, errors='replace')
   return error_message, log
   return error_message, log

+ 42 - 77
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -52,9 +52,9 @@ DEFAULT_USER = DEFAULT_USER.get()
 
 
 class HiveServerTable(Table):
 class HiveServerTable(Table):
   """
   """
-  We are parsing DESCRIBE EXTENDED text as the metastore API like GetColumns() misses most of the information.
-  Impala only supports a simple DESCRIBE.
+  We get the table details from a DESCRIBE FORMATTED.
   """
   """
+
   def __init__(self, table_results, table_schema, desc_results, desc_schema):
   def __init__(self, table_results, table_schema, desc_results, desc_schema):
     if beeswax_conf.THRIFT_VERSION.get() >= 7:
     if beeswax_conf.THRIFT_VERSION.get() >= 7:
       if not table_results.columns:
       if not table_results.columns:
@@ -69,99 +69,62 @@ class HiveServerTable(Table):
     self.desc_results = desc_results
     self.desc_results = desc_results
     self.desc_schema = desc_schema
     self.desc_schema = desc_schema
 
 
+    self.describe = HiveServerTTableSchema(self.desc_results, self.desc_schema).cols()
+
   @property
   @property
   def name(self):
   def name(self):
     return HiveServerTRow(self.table, self.table_schema).col('TABLE_NAME')
     return HiveServerTRow(self.table, self.table_schema).col('TABLE_NAME')
 
 
   @property
   @property
   def is_view(self):
   def is_view(self):
-    return HiveServerTRow(self.table, self.table_schema).col('TABLE_TYPE') == 'VIEW' # Used to be VIRTUAL_VIEW
+    return HiveServerTRow(self.table, self.table_schema).col('TABLE_TYPE') == 'VIEW'
 
 
   @property
   @property
   def partition_keys(self):
   def partition_keys(self):
-    describe = self.extended_describe
-    # Parses a list of: partitionKeys:[FieldSchema(name:baz, type:string, comment:null), FieldSchema(name:boom, type:string, comment:null)]
-    match = re.search('partitionKeys:\[([^\]]+)\]', describe)
-    if match is not None:
-      match = match.group(1)
-      return [PartitionKeyCompatible(*partition)
-          for partition in re.findall('FieldSchema\(name:(.+?), type:(.+?), comment:(.+?)\)', match)]
-    else:
+    try:
+      return [PartitionKeyCompatible(row['col_name'], row['data_type'], row['comment']) for row in self._get_partition_column()]
+    except:
       return []
       return []
 
 
   @property
   @property
   def path_location(self):
   def path_location(self):
     try:
     try:
-      describe = self.extended_describe
-      match = re.search('location:([^,]+)', describe)
-      if match is not None:
-        match = match.group(1)
-      return match
+      rows = self.describe
+      rows = [row for row in rows if row['col_name'].startswith('Location:')]
+      if rows:
+        return rows[0]['data_type']
     except:
     except:
-      # Impala does not have extended_describe
       return None
       return None
 
 
-  @property
-  def parameters(self):
-    # Parses a list of: parameters:{serialization.format=1}),... parameters:{numPartitions=2, EXTERNAL=TRUE}
-    describe = self.extended_describe
-    params = re.findall('parameters:\{([^\}]+?)\}', describe)
-    if params:
-      params_list = ', '.join(params).split(', ')
-      return dict([param.split('=')for param in params_list])
-    else:
-      return {}
-
   @property
   @property
   def cols(self):
   def cols(self):
-    cols = HiveServerTTableSchema(self.desc_results, self.desc_schema).cols()
+    rows = self.describe
     try:
     try:
-      end_cols_index = map(itemgetter('col_name'), cols).index('') # Truncate below extended describe
-      return cols[0:end_cols_index]
+      col_row_index = 2
+      end_cols_index = map(itemgetter('col_name'), rows[col_row_index:]).index('')
+      return rows[col_row_index:][:end_cols_index] + self._get_partition_column()
     except:
     except:
-      try:
-        # Spark SQL: does not have an empty line in extended describe
-        try:
-          end_cols_index = map(itemgetter('col_name'), cols).index('# Partition Information')
-        except:
-          end_cols_index = map(itemgetter('col_name'), cols).index('Detailed Table Information')
-        return cols[0:end_cols_index]
-      except:
-        # Impala: uses non extended describe and 'col' instead of 'col_name'
-        return cols
+      return rows
+
+  def _get_partition_column(self):
+    rows = self.describe
+    try:
+      col_row_index = map(itemgetter('col_name'), rows).index('# Partition Information') + 3
+      end_cols_index = map(itemgetter('col_name'), rows[col_row_index:]).index('')
+      return rows[col_row_index:][:end_cols_index]
+    except:
+      return []
 
 
   @property
   @property
   def comment(self):
   def comment(self):
     return HiveServerTRow(self.table, self.table_schema).col('REMARKS')
     return HiveServerTRow(self.table, self.table_schema).col('REMARKS')
 
 
-  @property
-  def extended_describe(self):
-    # Just keep rows after 'Detailed Table Information'
-    rows = HiveServerTTableSchema(self.desc_results, self.desc_schema).cols()
-    detailed_row_index = map(itemgetter('col_name'), rows).index('Detailed Table Information')
-    # Hack because of bad delimiter escaping in LazySimpleSerDe in HS2: parameters:{serialization.format=})
-    describe_text = rows[detailed_row_index]['data_type']
-    try:
-      # LazySimpleSerDe case, also add full next row
-      return describe_text + rows[detailed_row_index + 1]['col_name'] + rows[detailed_row_index + 1]['data_type']
-    except:
-      return describe_text
-
   @property
   @property
   def properties(self):
   def properties(self):
-    # Ugly but would need a recursive parsing to be clean
-    no_table = re.sub('\)$', '', re.sub('^Table\(', '', self.extended_describe))
-    properties = re.sub(', sd:StorageDescriptor\(cols.+?\]', '', no_table).split(', ')
-    props = []
-
-    for prop in properties:
-      key_val = prop.rsplit(':', 1)
-      if len(key_val) == 1:
-        key_val = key_val[0].rsplit('=', 1)
-      if len(key_val) == 2:
-        props.append(key_val)
-
-    return props
+    rows = self.describe
+    col_row_index = 2
+    end_cols_index = map(itemgetter('col_name'), rows[col_row_index:]).index('')
+    return rows[col_row_index + end_cols_index + 1:]
 
 
 
 
 class HiveServerTRowSet2:
 class HiveServerTRowSet2:
@@ -311,7 +274,7 @@ class HiveServerTTableSchema:
       cols = HiveServerTRowSet(self.columns, self.schema).cols(('name', 'type', 'comment'))
       cols = HiveServerTRowSet(self.columns, self.schema).cols(('name', 'type', 'comment'))
       for col in cols:
       for col in cols:
         col['col_name'] = col.pop('name')
         col['col_name'] = col.pop('name')
-        col['col_type'] = col.pop('type')
+        col['data_type'] = col.pop('type')
       return cols
       return cols
 
 
   def col(self, colName):
   def col(self, colName):
@@ -626,11 +589,7 @@ class HiveServerClient:
     table_results, table_schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT)
     table_results, table_schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT)
     self.close_operation(res.operationHandle)
     self.close_operation(res.operationHandle)
 
 
-    if self.query_server['server_name'] == 'impala':
-      # Impala does not supported extended
-      query = 'DESCRIBE %s' % table_name
-    else:
-      query = 'DESCRIBE EXTENDED %s' % table_name
+    query = 'DESCRIBE FORMATTED %s' % table_name
     (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000, orientation=TFetchOrientation.FETCH_NEXT)
     (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000, orientation=TFetchOrientation.FETCH_NEXT)
     self.close_operation(operation_handle)
     self.close_operation(operation_handle)
 
 
@@ -772,7 +731,7 @@ class HiveServerClient:
     else:
     else:
       max_rows = 1000 if max_parts <= 250 else max_parts
       max_rows = 1000 if max_parts <= 250 else max_parts
 
 
-    partitionTable = self.execute_query_statement('SHOW PARTITIONS %s' % table_name, max_rows=max_rows) # DB prefix supported since Hive 0.13
+    partitionTable = self.execute_query_statement('SHOW PARTITIONS %s.%s' % (database, table_name), max_rows=max_rows)
     return [PartitionValueCompatible(partition, table) for partition in partitionTable.rows()][-max_parts:]
     return [PartitionValueCompatible(partition, table) for partition in partitionTable.rows()][-max_parts:]
 
 
 
 
@@ -789,11 +748,17 @@ class HiveServerTableCompatible(HiveServerTable):
     self.desc_results = hive_table.desc_results
     self.desc_results = hive_table.desc_results
     self.desc_schema = hive_table.desc_schema
     self.desc_schema = hive_table.desc_schema
 
 
+    self.describe = HiveServerTTableSchema(self.desc_results, self.desc_schema).cols()
+
   @property
   @property
   def cols(self):
   def cols(self):
-    return [type('Col', (object,), {'name': col.get('col_name', '').strip(),
-                                    'type': col.get('data_type', col.get('col_type', '')).strip(), # Impala is col_type
-                                    'comment': col.get('comment', '').strip() if col.get('comment') else '', }) for col in HiveServerTable.cols.fget(self)]
+    return [
+        type('Col', (object,), {
+          'name': col.get('col_name', '').strip(),
+          'type': col.get('data_type', '').strip(),
+          'comment': col.get('comment', '').strip() if col.get('comment') else ''
+        }) for col in HiveServerTable.cols.fget(self)
+  ]
 
 
 
 
 class ResultCompatible:
 class ResultCompatible:

+ 110 - 117
apps/beeswax/src/beeswax/tests.py

@@ -996,7 +996,7 @@ for x in sys.stdin:
     hql = "SELECT foo, bar FROM test"
     hql = "SELECT foo, bar FROM test"
     resp = _make_query(self.client, hql, wait=True, local=False, max=180.0)
     resp = _make_query(self.client, hql, wait=True, local=False, max=180.0)
     resp = save_and_verify(resp, TARGET_FILE, overwrite=False, verify=False)
     resp = save_and_verify(resp, TARGET_FILE, overwrite=False, verify=False)
-    assert_true('-3' in resp.content)
+    assert_true('-3' in resp.content, resp.content)
     assert_true('already exists' in resp.content)
     assert_true('already exists' in resp.content)
 
 
     # Partition tables
     # Partition tables
@@ -1859,14 +1859,53 @@ ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=2);"""
 
 
 class MockHiveServerTable(HiveServerTable):
 class MockHiveServerTable(HiveServerTable):
 
 
-  def __init__(self, data):
-    self.path_location = data.get('path_location')
+  def __init__(self, describe=None):
+    if describe is not None:
+      self.describe = describe
+    else:
+      self.describe = [
+            {'comment': 'comment             ', 'col_name': '# col_name            ', 'data_type': 'data_type           '},
+            {'comment': None, 'col_name': '', 'data_type': None},
+            {'comment': '', 'col_name': 'foo', 'data_type': 'int'},
+            {'comment': '', 'col_name': 'bar', 'data_type': 'string'},
+            {'comment': None, 'col_name': '', 'data_type': None},
+            {'comment': None, 'col_name': '# Partition Information', 'data_type': None},
+            {'comment': 'comment             ', 'col_name': '# col_name            ', 'data_type': 'data_type           '},
+            {'comment': None, 'col_name': '', 'data_type': None},
+            {'comment': '', 'col_name': 'baz', 'data_type': 'string'},
+            {'comment': '', 'col_name': 'boom', 'data_type': 'string'},
+            {'comment': None, 'col_name': '', 'data_type': None},
+            {'comment': None, 'col_name': '# Detailed Table Information', 'data_type': None},
+            {'comment': None, 'col_name': 'Database:           ', 'data_type': 'default             '},
+            {'comment': None, 'col_name': 'Owner:              ', 'data_type': 'romain              '},
+            {'comment': None, 'col_name': 'CreateTime:         ', 'data_type': 'Wed Aug 13 13:39:53 PDT 2014'},
+            {'comment': None, 'col_name': 'LastAccessTime:     ', 'data_type': 'UNKNOWN             '},
+            {'comment': None, 'col_name': 'Protect Mode:       ', 'data_type': 'None                '},
+            {'comment': None, 'col_name': 'Retention:          ', 'data_type': '0                   '},
+            {'comment': None, 'col_name': 'Location:           ', 'data_type': 'hdfs://localhost:8020/user/hive/warehouse/test_partitions'},
+            {'comment': None, 'col_name': 'Table Type:         ', 'data_type': 'MANAGED_TABLE       '},
+            {'comment': None, 'col_name': 'Table Parameters:', 'data_type': None},
+            {'comment': '1407962393          ', 'col_name': '', 'data_type': 'transient_lastDdlTime'},
+            {'comment': None, 'col_name': '', 'data_type': None},
+            {'comment': None, 'col_name': '# Storage Information', 'data_type': None},
+            {'comment': None, 'col_name': 'SerDe Library:      ', 'data_type': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'},
+            {'comment': None, 'col_name': 'InputFormat:        ', 'data_type': 'org.apache.hadoop.mapred.TextInputFormat'},
+            {'comment': None, 'col_name': 'OutputFormat:       ', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'},
+            {'comment': None, 'col_name': 'Compressed:         ', 'data_type': 'No                  '},
+            {'comment': None, 'col_name': 'Num Buckets:        ', 'data_type': '-1                  '},
+            {'comment': None, 'col_name': 'Bucket Columns:     ', 'data_type': '[]                  '},
+            {'comment': None, 'col_name': 'Sort Columns:       ', 'data_type': '[]                  '},
+            {'comment': None, 'col_name': 'Storage Desc Params:', 'data_type': None},
+            {'comment': '\\t                  ', 'col_name': '', 'data_type': 'field.delim         '},
+            {'comment': '\\n                  ', 'col_name': '', 'data_type': 'line.delim          '},
+            {'comment': '\\t                  ', 'col_name': '', 'data_type': 'serialization.format'}
+        ]
 
 
 
 
 class TestHiveServer2API():
 class TestHiveServer2API():
 
 
   def test_parsing_partition_values(self):
   def test_parsing_partition_values(self):
-    table = MockHiveServerTable({'path_location': '/my/table'})
+    table = MockHiveServerTable()
 
 
     value = PartitionValueCompatible(['datehour=2013022516'], table)
     value = PartitionValueCompatible(['datehour=2013022516'], table)
     assert_equal(['2013022516'], value.values)
     assert_equal(['2013022516'], value.values)
@@ -1874,120 +1913,74 @@ class TestHiveServer2API():
     value = PartitionValueCompatible(['month=2011-07/dt=2011-07-01/hr=12'], table)
     value = PartitionValueCompatible(['month=2011-07/dt=2011-07-01/hr=12'], table)
     assert_equal(['2011-07', '2011-07-01', '12'], value.values)
     assert_equal(['2011-07', '2011-07-01', '12'], value.values)
 
 
-  def test_table_properties(self):
-    table = MockHiveServerTable({})
-    prev_extended_describe = getattr(MockHiveServerTable, 'extended_describe')
+  def test_hiveserver_table(self):
+    table = MockHiveServerTable()
+
+    assert_equal([
+        {'comment': None, 'col_name': '# Partition Information', 'data_type': None},
+        {'comment': 'comment             ', 'col_name': '# col_name            ', 'data_type': 'data_type           '},
+        {'comment': None, 'col_name': '', 'data_type': None},
+        {'comment': '', 'col_name': 'baz', 'data_type': 'string'},
+        {'comment': '', 'col_name': 'boom', 'data_type': 'string'},
+        {'comment': None, 'col_name': '', 'data_type': None},
+        {'comment': None, 'col_name': '# Detailed Table Information', 'data_type': None},
+        {'comment': None, 'col_name': 'Database:           ', 'data_type': 'default             '},
+        {'comment': None, 'col_name': 'Owner:              ', 'data_type': 'romain              '},
+        {'comment': None, 'col_name': 'CreateTime:         ', 'data_type': 'Wed Aug 13 13:39:53 PDT 2014'},
+        {'comment': None, 'col_name': 'LastAccessTime:     ', 'data_type': 'UNKNOWN             '},
+        {'comment': None, 'col_name': 'Protect Mode:       ', 'data_type': 'None                '},
+        {'comment': None, 'col_name': 'Retention:          ', 'data_type': '0                   '},
+        {'comment': None, 'col_name': 'Location:           ', 'data_type': 'hdfs://localhost:8020/user/hive/warehouse/test_partitions'},
+        {'comment': None, 'col_name': 'Table Type:         ', 'data_type': 'MANAGED_TABLE       '},
+        {'comment': None, 'col_name': 'Table Parameters:', 'data_type': None},
+        {'comment': '1407962393          ', 'col_name': '', 'data_type': 'transient_lastDdlTime'},
+        {'comment': None, 'col_name': '', 'data_type': None},
+        {'comment': None, 'col_name': '# Storage Information', 'data_type': None},
+        {'comment': None, 'col_name': 'SerDe Library:      ', 'data_type': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'},
+        {'comment': None, 'col_name': 'InputFormat:        ', 'data_type': 'org.apache.hadoop.mapred.TextInputFormat'},
+        {'comment': None, 'col_name': 'OutputFormat:       ', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'},
+        {'comment': None, 'col_name': 'Compressed:         ', 'data_type': 'No                  '},
+        {'comment': None, 'col_name': 'Num Buckets:        ', 'data_type': '-1                  '},
+        {'comment': None, 'col_name': 'Bucket Columns:     ', 'data_type': '[]                  '},
+        {'comment': None, 'col_name': 'Sort Columns:       ', 'data_type': '[]                  '},
+        {'comment': None, 'col_name': 'Storage Desc Params:', 'data_type': None},
+        {'comment': '\\t                  ', 'col_name': '', 'data_type': 'field.delim         '},
+        {'comment': '\\n                  ', 'col_name': '', 'data_type': 'line.delim          '},
+        {'comment': '\\t                  ', 'col_name': '', 'data_type': 'serialization.format'}],
+            table.properties)
+
+    assert_equal('hdfs://localhost:8020/user/hive/warehouse/test_partitions', table.path_location)
+
+    assert_equal([
+      {'col_name': 'foo', 'comment': '', 'data_type': 'int'},
+      {'col_name': 'bar', 'comment': '', 'data_type': 'string'},
+      {'col_name': 'baz', 'comment': '', 'data_type': 'string'},
+      {'col_name': 'boom', 'comment': '', 'data_type': 'string'}], table.cols)
+
+    assert_equal([PartitionKeyCompatible('baz', 'string', ''),
+                  PartitionKeyCompatible('boom', 'string', '')
+                 ], table.partition_keys)
+
+
+  def test_hiveserver_table_partition_keys(self):
+    describe = [
+        {'comment': None, 'col_name': '# Partition Information', 'data_type': None},
+        {'comment': 'comment             ', 'col_name': '# col_name            ', 'data_type': 'data_type           '},
+        {'comment': None, 'col_name': '', 'data_type': None},
+        {'comment': '', 'col_name': 'dt', 'data_type': 'string'},
+        {'comment': '', 'col_name': 'country', 'data_type': 'string'},
+        {'comment': 'this, has extra: sigils', 'col_name': 'decimal', 'data_type': 'decimal(9, 7)'},
+        {'comment': '', 'col_name': 'complex', 'data_type': 'UNIONTYPE<int, double, array<string>, struct<a:int,b:string>>'},
+        {'comment': None, 'col_name': '', 'data_type': None}
+    ]
+    table = MockHiveServerTable(describe)
+
+    assert_equal([PartitionKeyCompatible('dt', 'string', ''),
+                  PartitionKeyCompatible('country', 'string', ''),
+                  PartitionKeyCompatible('decimal', 'decimal(9, 7)', 'this, has extra: sigils'),
+                  PartitionKeyCompatible('complex', 'UNIONTYPE<int, double, array<string>, struct<a:int,b:string>>', ''),
+                 ], table.partition_keys)
 
 
-    try:
-      extended_describe = (
-        'Table('
-          'tableName:page_view, '
-          'dbName:default, '
-          'owner:romain, '
-          'createTime:1360732885, '
-          'lastAccessTime:0, '
-          'retention:0, '
-          'sd:StorageDescriptor('
-            'cols:['
-              'FieldSchema(name:viewtime, type:int, comment:null), '
-              'FieldSchema(name:userid, type:bigint, comment:null), '
-              'FieldSchema(name:page_url, type:string, comment:null), '
-              'FieldSchema(name:referrer_url, type:string, comment:null), '
-              'FieldSchema(name:ip, type:string, comment:IP Address of the User), '
-              'FieldSchema(name:dt, type:string, comment:null), '
-              'FieldSchema(name:country, type:string, comment:null)'
-            '], '
-            'location:hdfs://localhost:8020/user/hive/warehouse/page_view, '
-            'inputFormat:org.apache.hadoop.mapred.TextInputFormat, '
-            'outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, '
-            'compressed:false, '
-            'numBuckets:-1, '
-            'serdeInfo:SerDeInfo('
-              'name:null, '
-              'serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, '
-              'parameters:{serialization.format=1}'
-            '), '
-            'bucketCols:[], '
-            'sortCols:[], '
-            'parameters:{}, '
-            'skewedInfo:SkewedInfo('
-              'skewedColNames:[], '
-              'skewedColValues:[], '
-              'skewedColValueLocationMaps:{}'
-            '), '
-            'storedAsSubDirectories:false'
-          '), '
-          'partitionKeys:['
-            'FieldSchema(name:dt, type:string, comment:null), '
-            'FieldSchema(name:country, type:string, comment:null), '
-            'FieldSchema(name:decimal, type:decimal(9, 7), comment:this, has extra: sigils), '
-            'FieldSchema(name:complex, type:UNIONTYPE<int, double, array<string>, struct<a:int,b:string>>, comment:null), '
-          '], '
-          'parameters:{'
-            'numPartitions=0, '
-            'numFiles=1, '
-            'transient_lastDdlTime=1360732885, '
-            'comment=This is the page view table'
-          '}, '
-          'viewOriginalText:null, '
-          'viewExpandedText:null, '
-          'tableType:MANAGED_TABLE'
-        ')'
-      )
-      setattr(table, 'extended_describe', extended_describe)
-
-      assert_equal([['tableName', 'page_view'],
-                    ['dbName', 'default'],
-                    ['owner', 'romain'],
-                    ['createTime', '1360732885'],
-                    ['lastAccessTime', '0'],
-                    ['retention', '0'],
-                    ['location:hdfs://localhost', '8020/user/hive/warehouse/page_view'],
-                    ['inputFormat', 'org.apache.hadoop.mapred.TextInputFormat'],
-                    ['outputFormat', 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'],
-                    ['compressed', 'false'],
-                    ['numBuckets', '-1'],
-                    ['serdeInfo:SerDeInfo(name', 'null'],
-                    ['serializationLib', 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'],
-                    ['parameters', '{serialization.format=1})'],
-                    ['bucketCols', '[]'],
-                    ['sortCols', '[]'],
-                    ['parameters', '{}'],
-                    ['skewedInfo:SkewedInfo(skewedColNames', '[]'],
-                    ['skewedColValues', '[]'],
-                    ['skewedColValueLocationMaps', '{})'],
-                    ['storedAsSubDirectories', 'false)'],
-                    ['partitionKeys:[FieldSchema(name', 'dt'],
-                    ['type', 'string'],
-                    ['comment', 'null)'],
-                    ['FieldSchema(name', 'country'],
-                    ['type', 'string'],
-                    ['comment', 'null)'],
-                    ['FieldSchema(name', 'decimal'],
-                    ['type', 'decimal(9'],
-                    ['comment', 'this'],
-                    ['has extra', ' sigils)'],
-                    ['FieldSchema(name', 'complex'],
-                    ['type', 'UNIONTYPE<int'],
-                    ['struct<a:int,b', 'string>>'],
-                    ['comment', 'null)'],
-                    ['parameters', '{numPartitions=0'],
-                    ['numFiles', '1'],
-                    ['transient_lastDdlTime', '1360732885'],
-                    ['comment', 'This is the page view table}'],
-                    ['viewOriginalText', 'null'],
-                    ['viewExpandedText', 'null'],
-                    ['tableType', 'MANAGED_TABLE']
-                  ],
-                  table.properties)
-
-      assert_equal([PartitionKeyCompatible('dt', 'string', 'null'),
-                    PartitionKeyCompatible('country', 'string', 'null'),
-                    PartitionKeyCompatible('decimal', 'decimal(9, 7)', 'this, has extra: sigils'),
-                    PartitionKeyCompatible('complex', 'UNIONTYPE<int, double, array<string>, struct<a:int,b:string>>', 'null'),
-                   ], table.partition_keys)
-    finally:
-      setattr(table, 'extended_describe', prev_extended_describe)
 
 
   def test_column_format_values_nulls(self):
   def test_column_format_values_nulls(self):
     data = [1, 1, 1]
     data = [1, 1, 1]

+ 7 - 5
apps/metastore/src/metastore/templates/describe_table.mako

@@ -103,12 +103,12 @@ ${ components.menubar() }
 
 
             <div class="tab-content">
             <div class="tab-content">
               <div class="active tab-pane" id="columns">
               <div class="active tab-pane" id="columns">
-                ${column_table(table.cols)}
+                ${ column_table(table.cols) }
               </div>
               </div>
 
 
               % if table.partition_keys:
               % if table.partition_keys:
               <div class="tab-pane" id="partitionColumns">
               <div class="tab-pane" id="partitionColumns">
-                ${column_table(table.partition_keys)}
+                ${ column_table(table.partition_keys) }
               </div>
               </div>
               % endif
               % endif
 
 
@@ -156,13 +156,15 @@ ${ components.menubar() }
                     <tr>
                     <tr>
                       <th>${ _('Name') }</th>
                       <th>${ _('Name') }</th>
                       <th>${ _('Value') }</th>
                       <th>${ _('Value') }</th>
+                      <th>${ _('Comment') }</th>
                     </tr>
                     </tr>
                   </thead>
                   </thead>
                   <tbody>
                   <tbody>
-                    % for name, value in table.properties:
+                    % for prop in table.properties:
                       <tr>
                       <tr>
-                        <td>${ smart_unicode(name) }</td>
-                        <td>${ smart_unicode(value) }</td>
+                        <td>${ smart_unicode(prop['col_name']) }</td>
+                        <td>${ smart_unicode(prop['data_type']) if prop['data_type'] else '' }</td>
+                        <td>${ smart_unicode(prop['comment']) if prop['comment'] else '' }&nbsp;</td>
                       </tr>
                       </tr>
                      % endfor
                      % endfor
                   </tbody>
                   </tbody>

+ 1 - 1
apps/metastore/src/metastore/tests.py

@@ -84,7 +84,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     # And have detail
     # And have detail
     response = self.client.get("/metastore/table/default/test")
     response = self.client.get("/metastore/table/default/test")
     assert_true("foo" in response.content)
     assert_true("foo" in response.content)
-    assert_true("serdeInfo:SerDeInfo" in response.content, response.content)
+    assert_true("SerDe Library" in response.content, response.content)
 
 
     # Remember the number of history items. Use a generic fragment 'test' to pass verification.
     # Remember the number of history items. Use a generic fragment 'test' to pass verification.
     history_cnt = verify_history(self.client, fragment='test')
     history_cnt = verify_history(self.client, fragment='test')