Browse Source

[Importer] Update Iceberg table syntax for Impala (#3887)

Ayush Goyal 1 year ago
parent
commit
a8a85851ee

+ 1 - 0
desktop/libs/indexer/src/indexer/indexers/sql.py

@@ -253,6 +253,7 @@ class SQLIndexer(object):
             extra_create_properties = "STORED BY ICEBERG\n"
           elif source_type == 'impala':
             file_format = 'ICEBERG'
+            external = False
         extra_create_properties += 'STORED AS %(file_format)s' % {'file_format': file_format}
         if is_transactional:
           extra_create_properties += "\nTBLPROPERTIES('transactional'='true', 'transactional_properties'='%s')" % \

+ 98 - 4
desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -1279,7 +1279,7 @@ TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 
 
 @pytest.mark.django_db
-def test_generate_create_iceberg_table():
+def test_generate_create_hive_iceberg_table():
   source = json.loads(
     '''{"sourceType": "hive", "name":"","sample":[["Bank Of America","3000000.0","US","Miami","37.6801986694",'''
     '''"-121.92150116"],["Citi Bank","2800000.0","US","Richmond","37.5242004395","-77.4932022095"],["Deutsche Bank","2600000.0","US",'''
@@ -1336,7 +1336,7 @@ def test_generate_create_iceberg_table():
     '''"text","name":"Text"},{"value":"parquet","name":"Parquet"},{"value":"kudu","name":"Kudu"},{"value":"csv","name":"Csv"},'''
     '''{"value":"avro","name":"Avro"},{"value":"json","name":"Json"},{"value":"regexp","name":"Regexp"},{"value":"orc",'''
     '''"name":"ORC"}],"partitionColumns":[],"kuduPartitionColumns":[],"primaryKeys":[],"primaryKeyObjects":[],"importData":true,'''
-    '''"isIceberg":true,"useCopy":false,"useDefaultLocation":true,"nonDefaultLocation":"/user/hue/data/query-hive-360.csv",'''
+    '''"isIceberg":true,"useCopy":false,"useDefaultLocation":false,"nonDefaultLocation":"/user/hue/data/query-hive-360.csv",'''
     '''"hasHeader":true,"useCustomDelimiters":false,"customFieldDelimiter":",","customCollectionDelimiter":"\\\\002",'''
     '''"customMapDelimiter":"\\\\003","customRegexp":""}'''
   )
@@ -1346,7 +1346,6 @@ def test_generate_create_iceberg_table():
 
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
 
-  print(sql)
   assert '''USE default;''' in sql, sql
 
   statement = '''CREATE EXTERNAL TABLE IF NOT EXISTS `default`.`hue__tmp_parquet_table`
@@ -1366,7 +1365,7 @@ TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
   assert statement in sql, sql
 
   assert (
-    '''CREATE TABLE `default`.`parquet_table`
+    '''CREATE EXTERNAL TABLE `default`.`parquet_table`
         STORED BY ICEBERG
 STORED AS parquet
         AS SELECT *
@@ -1378,6 +1377,101 @@ STORED AS parquet
   assert '''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;''' in sql, sql
 
 
+@pytest.mark.django_db
+def test_generate_create_impala_iceberg_table():
+  source = json.loads(
+    '''{"sourceType": "impala", "name":"","sample":[["Bank Of America","3000000.0","US","Miami","37.6801986694",'''
+    '''"-121.92150116"],["Citi Bank","2800000.0","US","Richmond","37.5242004395","-77.4932022095"],["Deutsche Bank","2600000.0","US",'''
+    '''"Corpus Christi","40.7807998657","-73.9772033691"],["Thomson Reuters","2400000.0","US","Albany","35.7976989746",'''
+    '''"-78.6252975464"],'''
+    '''["OpenX","2200000.0","US","Des Moines","40.5411987305","-119.586898804"]],"sampleCols":[{"operations":[],"comment":"",'''
+    '''"nested":[],'''
+    '''"name":"acct_client","level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,'''
+    '''"partitionValue":"","multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],'''
+    '''"comment":"","nested":[],"name":"tran_amount","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
+    '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"double",'''
+    '''"showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"tran_country_cd","level":0,"keyType":'''
+    '''"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"","multiValued":false,'''
+    '''"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"vrfcn_city",'''
+    '''"level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"",'''
+    '''"multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],'''
+    '''"name":"vrfcn_city_lat","level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,'''
+    '''"length":100,'''
+    '''"partitionValue":"","multiValued":false,"unique":false,"type":"double","showProperties":false,"scale":0},{"operations":[],'''
+    '''"comment":"","nested":[],"name":"vrfcn_city_lon","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
+    '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"double","showProperties":false,'''
+    '''"scale":0}],"inputFormat":"file","inputFormatsAll":[{"value":"file","name":"File"},{"value":"manual","name":"Manually"},'''
+    '''{"value":"query","name":"SQL Query"},{"value":"table","name":"Table"}],"inputFormatsManual":[{"value":"manual","name":'''
+    '''"Manually"}],"inputFormats":[{"value":"file","name":"File"},{"value":"manual","name":"Manually"},{"value":"query","name":'''
+    '''"SQL Query"},{"value":"table","name":"Table"}],"path":"/user/hue/data/query-impala-360.csv","isObjectStore":false,"table":"",'''
+    '''"tableName":"","databaseName":"default","apiHelperType":"impala","query":"","draggedQuery":"","format":{"type":"csv",'''
+    '''"fieldSeparator":",","recordSeparator":"\\n","quoteChar":"\\"","hasHeader":true,"status":0},"show":true,"defaultName":'''
+    '''"default.query-impala-360"}'''
+  )
+  destination = json.loads(
+    '''{"isTransactional": false, "isInsertOnly": false, "sourceType": "impala", "name":"default.parquet_table"'''
+    ''',"apiHelperType":"impala","description":"","outputFormat":"table","outputFormatsList":[{"name":"Table","value":"table"},'''
+    '''{"name":"Solr index","value":"index"},{"name":"File","value":"file"},{"name":"Database","value":"database"}],'''
+    '''"outputFormats":[{"name":"Table","value":"table"},{"name":"Solr index","value":"index"}],"columns":[{"operations":[],'''
+    '''"comment":"","nested":[],"name":"acct_client","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
+    '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"string","showProperties":'''
+    '''false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"tran_amount","level":0,"keyType":"string","required":false,'''
+    '''"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":'''
+    '''"double","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"tran_country_cd","level":0,'''
+    '''"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"",'''
+    '''"multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":'''
+    '''[],"name":"vrfcn_city","level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":'''
+    '''100,"partitionValue":"","multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],'''
+    '''"comment":"","nested":[],"name":"vrfcn_city_lat","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
+    '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"double","showProperties":'''
+    '''false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"vrfcn_city_lon","level":0,"keyType":"string","required":'''
+    '''false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,'''
+    '''"type":"double","showProperties":false,"scale":0}],"bulkColumnNames":"acct_client,tran_amount,tran_country_cd,vrfcn_city,'''
+    '''vrfcn_city_lat,vrfcn_city_lon","showProperties":false,"isTargetExisting":false,"isTargetChecking":false,"existingTargetUrl":'''
+    '''"","tableName":"parquet_table","databaseName":"default","tableFormat":"parquet","KUDU_DEFAULT_RANGE_PARTITION_COLUMN":'''
+    '''{"values":[{"value":""}],"name":"VALUES","lower_val":0,"include_lower_val":"<=","upper_val":1,"include_upper_val":"<="},'''
+    '''"KUDU_DEFAULT_PARTITION_COLUMN":{"columns":[],"range_partitions":[{"values":[{"value":""}],"name":"VALUES","lower_val":0,'''
+    '''"include_lower_val":"<=","upper_val":1,"include_upper_val":"<="}],"name":"HASH","int_val":16},"tableFormats":[{"value":'''
+    '''"text","name":"Text"},{"value":"parquet","name":"Parquet"},{"value":"kudu","name":"Kudu"},{"value":"csv","name":"Csv"},'''
+    '''{"value":"avro","name":"Avro"},{"value":"json","name":"Json"},{"value":"regexp","name":"Regexp"},{"value":"orc",'''
+    '''"name":"ORC"}],"partitionColumns":[],"kuduPartitionColumns":[],"primaryKeys":[],"primaryKeyObjects":[],"importData":true,'''
+    '''"isIceberg":true,"useCopy":false,"useDefaultLocation":false,"nonDefaultLocation":"/user/hue/data/query-impala-360.csv",'''
+    '''"hasHeader":true,"useCustomDelimiters":false,"customFieldDelimiter":",","customCollectionDelimiter":"\\\\002",'''
+    '''"customMapDelimiter":"\\\\003","customRegexp":""}'''
+  )
+
+  path = {'isDir': False, 'split': ('/user/hue/data', 'query-impala-360.csv'), 'listdir': ['/user/hue/data']}
+  request = MockRequest(fs=MockFs(path=path))
+
+  sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
+
+  assert '''USE default;''' in sql, sql
+
+  statement = '''CREATE EXTERNAL TABLE IF NOT EXISTS `default`.`hue__tmp_parquet_table`
+(
+  `acct_client` string ,
+  `tran_amount` double ,
+  `tran_country_cd` string ,
+  `vrfcn_city` string ,
+  `vrfcn_city_lat` double ,
+  `vrfcn_city_lon` double ) ROW FORMAT   DELIMITED
+    FIELDS TERMINATED BY ','
+  STORED AS TextFile LOCATION '/user/hue/data'
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
+;'''
+  assert statement in sql, sql
+
+  assert (
+    '''CREATE TABLE `default`.`parquet_table`
+        STORED AS ICEBERG
+        AS SELECT *
+        FROM `default`.`hue__tmp_parquet_table`;'''
+    in sql
+  ), sql
+
+  assert '''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;''' in sql, sql
+
+
 @pytest.mark.django_db
 def test_generate_create_orc_table_transactional():
   source = json.loads(