فهرست منبع

HUE-9191 [hive] API to retrieve table Foreign Keys

Romain 5 سال پیش
والد
کامیت
3ffc3de8ac
2فایلهای تغییر یافته به همراه78 افزوده شده و 4 حذف شده
  1. 23 4
      apps/beeswax/src/beeswax/server/hive_server2_lib.py
  2. 55 0
      apps/beeswax/src/beeswax/server/hive_server2_lib_tests.py

+ 23 - 4
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -136,11 +136,11 @@ class HiveServerTable(Table):
       # Not partitioned
       return []
 
-  @property
-  def primary_keys(self):
+  def _parse_keys(self, key_name):
     rows = self.describe
+
     try:
-      col_row_index = list(map(itemgetter('col_name'), rows)).index('# Primary Key') + 3
+      col_row_index = list(map(itemgetter('col_name'), rows)).index(key_name) + 3
       try:
         end_cols_index = list(map(itemgetter('col_name'), rows[col_row_index:])).index('')
         keys = rows[col_row_index:][:end_cols_index]
@@ -150,7 +150,26 @@ class HiveServerTable(Table):
       # No info (e.g. IMPALA-8291)
       keys = []
 
-    return [PartitionKeyCompatible(row['data_type'].strip(), 'NULL', row['comment']) for row in keys]
+    return keys
+
+  @property
+  def primary_keys(self):
+    # Note: Thrift has GetPrimaryKeys() API
+    return [
+      PartitionKeyCompatible(row['data_type'].strip(), 'NULL', row['comment']) for row in self._parse_keys(key_name='# Primary Key')
+    ]
+
+  @property
+  def foreign_keys(self):
+    # Note: Thrift has GetCrossReference() API
+    return [
+      PartitionKeyCompatible(
+        row['data_type'].strip().split(':', 1)[1],  # from: Column Name:head
+        row['col_name'].strip().split(':', 1)[1],  # to: Parent Column Name:default.persons.id
+        row['comment']
+      )
+      for row in self._parse_keys(key_name='# Foreign Key')
+    ]
 
   @property
   def comment(self):

+ 55 - 0
apps/beeswax/src/beeswax/server/hive_server2_lib_tests.py

@@ -416,6 +416,61 @@ class TestHiveServerTable():
       assert_equal(table.primary_keys[1].comment, 'NULL')
 
 
+  def test_foreign_keys_hive(self):
+
+      table_results = Mock()
+      table_schema = Mock()
+      desc_results = Mock(
+        columns=[
+          # Dump of `DESCRIBE FORMATTED table`
+          Mock(
+            stringVal=Mock(values=['# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', '', '', '# Constraints', '',
+                '# Primary Key', 'Table:', 'Constraint Name:', 'Column Name:', '',
+                '# Foreign Key', 'Table:', 'Constraint Name:', 'Parent Column Name:default.persons.id', ''
+              ],
+              nulls=''
+            )
+          ),
+          Mock(
+            stringVal=Mock(values=['data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', 'NULL', 'NULL', 'NULL',
+                'NULL', 'default.pk', 'pk_165400321_1572980510006_0', 'id1 ', 'NULL',
+                'NULL', 'default.businessunit', 'fk', 'Column Name:head', 'NULL'
+              ],
+              nulls=''
+            )
+          ),
+          Mock(
+            stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
+                'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
+                'NULL', 'NULL', 'NULL', 'Key Sequence:1', 'NULL',
+              ],
+              nulls=''
+            )
+          ),
+        ]
+      )
+      desc_schema = Mock(
+        columns=[
+          Mock(columnName='col_name'),
+          Mock(columnName='data_type'),
+          Mock(columnName='comment')
+        ]
+      )
+
+      table = HiveServerTable(
+        table_results=table_results,
+        table_schema=table_schema,
+        desc_results=desc_results,
+        desc_schema=desc_schema
+      )
+
+      assert_equal(len(table.foreign_keys), 1)
+      assert_equal(table.foreign_keys[0].name, 'head')  # 'from' column
+      assert_equal(table.foreign_keys[0].type, 'default.persons.id')  # 'to' column
+      assert_equal(table.foreign_keys[0].comment, 'NULL')
+
+
+
 class TestSessionManagement():
 
   def setUp(self):