浏览代码

[beeswax] Support Spark SQL table metadata format

It is slightly different than HiveServer2.
Romain Rigaux 11 年之前
父节点
当前提交
9419cbc8a2
共有 1 个文件被更改,包括 12 次插入3 次删除
  1. 12 3
      apps/beeswax/src/beeswax/server/hive_server2_lib.py

+ 12 - 3
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -44,6 +44,7 @@ LOG = logging.getLogger(__name__)
 IMPALA_RESULTSET_CACHE_SIZE = 'impala.resultset.cache.size'
 IMPALA_RESULTSET_CACHE_SIZE = 'impala.resultset.cache.size'
 DEFAULT_USER = DEFAULT_USER.get()
 DEFAULT_USER = DEFAULT_USER.get()
 
 
+
 class HiveServerTable(Table):
 class HiveServerTable(Table):
   """
   """
   We are parsing DESCRIBE EXTENDED text as the metastore API like GetColumns() misses most of the information.
   We are parsing DESCRIBE EXTENDED text as the metastore API like GetColumns() misses most of the information.
@@ -107,8 +108,16 @@ class HiveServerTable(Table):
       end_cols_index = map(itemgetter('col_name'), cols).index('') # Truncate below extended describe
       end_cols_index = map(itemgetter('col_name'), cols).index('') # Truncate below extended describe
       return cols[0:end_cols_index]
       return cols[0:end_cols_index]
     except:
     except:
-      # Impala use non extended describe and 'col' instead of 'col_name'
-      return cols
+      try:
+        # Spark SQL: does not have an empty line in extended describe
+        try:
+          end_cols_index = map(itemgetter('col_name'), cols).index('# Partition Information')
+        except:
+          end_cols_index = map(itemgetter('col_name'), cols).index('Detailed Table Information')
+        return cols[0:end_cols_index]
+      except:
+        # Impala: uses non extended describe and 'col' instead of 'col_name'
+        return cols
 
 
   @property
   @property
   def comment(self):
   def comment(self):
@@ -651,7 +660,7 @@ class HiveServerTableCompatible(HiveServerTable):
   def cols(self):
   def cols(self):
     return [type('Col', (object,), {'name': col.get('col_name', '').strip(),
     return [type('Col', (object,), {'name': col.get('col_name', '').strip(),
                                     'type': col.get('data_type', col.get('col_type', '')).strip(), # Impala is col_type
                                     'type': col.get('data_type', col.get('col_type', '')).strip(), # Impala is col_type
-                                    'comment': col.get('comment', '').strip(), }) for col in HiveServerTable.cols.fget(self)]
+                                    'comment': col.get('comment', '').strip() if col.get('comment') else '', }) for col in HiveServerTable.cols.fget(self)]
 
 
 
 
 class ResultCompatible:
 class ResultCompatible: