Преглед изворни кода

[beeswax] Support Spark SQL table metadata format

It is slightly different than HiveServer2.
Romain Rigaux пре 11 година
родитељ
комит
9419cbc8a2
1 измењених фајлова са 12 додато и 3 уклоњено
  1. 12 3
      apps/beeswax/src/beeswax/server/hive_server2_lib.py

+ 12 - 3
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -44,6 +44,7 @@ LOG = logging.getLogger(__name__)
 IMPALA_RESULTSET_CACHE_SIZE = 'impala.resultset.cache.size'
 DEFAULT_USER = DEFAULT_USER.get()
 
+
 class HiveServerTable(Table):
   """
   We are parsing DESCRIBE EXTENDED text as the metastore API like GetColumns() misses most of the information.
@@ -107,8 +108,16 @@ class HiveServerTable(Table):
       end_cols_index = map(itemgetter('col_name'), cols).index('') # Truncate below extended describe
       return cols[0:end_cols_index]
     except:
-      # Impala use non extended describe and 'col' instead of 'col_name'
-      return cols
+      try:
+        # Spark SQL: does not have an empty line in extended describe
+        try:
+          end_cols_index = map(itemgetter('col_name'), cols).index('# Partition Information')
+        except:
+          end_cols_index = map(itemgetter('col_name'), cols).index('Detailed Table Information')
+        return cols[0:end_cols_index]
+      except:
+        # Impala: uses non extended describe and 'col' instead of 'col_name'
+        return cols
 
   @property
   def comment(self):
@@ -651,7 +660,7 @@ class HiveServerTableCompatible(HiveServerTable):
   def cols(self):
     return [type('Col', (object,), {'name': col.get('col_name', '').strip(),
                                     'type': col.get('data_type', col.get('col_type', '')).strip(), # Impala is col_type
-                                    'comment': col.get('comment', '').strip(), }) for col in HiveServerTable.cols.fget(self)]
+                                    'comment': col.get('comment', '').strip() if col.get('comment') else '', }) for col in HiveServerTable.cols.fget(self)]
 
 
 class ResultCompatible: