Răsfoiți Sursa

HUE-9240 [hive] Do no skip first table column on LLAP upstream

Another corner case: LLAP has headers but no following blank line
Romain 5 ani în urmă
părinte
comite
e3dfba6478

+ 7 - 4
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -114,12 +114,15 @@ class HiveServerTable(Table):
     col_row_index = 0
     try:
       cols = [col.strip() for col in map(itemgetter('col_name'), rows[col_row_index:])]
-      if cols[0] == '# col_name': # Hive MR/Impala have headers and one blank line, Hive Tez has nothing
-        col_row_index = 2
-        cols = cols[2:]
+      # Hive MR/Impala have headers and one blank line, Hive Tez has nothing, Hive LLAP upstream has headers and no blank line
+      if cols[0] == '# col_name':
+        col_row_index = 1
+        if not cols[1]:
+          col_row_index += 1
+        cols = cols[col_row_index:]
       end_cols_index = cols.index('')
       return rows[col_row_index:][:end_cols_index] + self._get_partition_columns()
-    except ValueError: # DESCRIBE on nested columns does not always contain additional rows beyond cols
+    except ValueError:  # DESCRIBE on nested columns does not always contain additional rows beyond cols
       return rows[col_row_index:]
     except:
       return rows

+ 34 - 0
apps/beeswax/src/beeswax/server/hive_server2_lib_tests.py

@@ -355,6 +355,40 @@ class TestHiveServerTable():
       assert_equal(table.cols[3], {'col_name': 'salary', 'data_type': 'int', 'comment': ''})
 
 
+  def test_cols_hive_llap_upstream(self):
+
+    table_results = Mock()
+    table_schema = Mock()
+    desc_results = Mock(
+      columns=[
+        # No empty line after headers
+        Mock(stringVal=Mock(values=['# col_name', 'code', 'description', 'total_emp', 'salary', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', ], nulls='')),
+        Mock(stringVal=Mock(values=['data_type', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', ], nulls='')),
+        Mock(stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', ], nulls='')),
+      ]
+    )
+    desc_schema = Mock(
+      columns=[
+        Mock(columnName='col_name'),
+        Mock(columnName='data_type'),
+        Mock(columnName='comment')
+      ]
+    )
+
+    table = HiveServerTable(
+      table_results=table_results,
+      table_schema=table_schema,
+      desc_results=desc_results,
+      desc_schema=desc_schema
+    )
+
+    assert_equal(len(table.cols), 4)
+    assert_equal(table.cols[0], {'col_name': 'code', 'data_type': 'string', 'comment': 'NULL'})
+    assert_equal(table.cols[1], {'col_name': 'description', 'data_type': 'string', 'comment': 'NULL'})
+    assert_equal(table.cols[2], {'col_name': 'total_emp', 'data_type': 'int', 'comment': 'NULL'})
+    assert_equal(table.cols[3], {'col_name': 'salary', 'data_type': 'int', 'comment': 'NULL'})
+
+
   def test_partition_keys_impala(self):
 
       table_results = Mock()