|
@@ -1279,7 +1279,7 @@ TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@pytest.mark.django_db
|
|
|
-def test_generate_create_iceberg_table():
|
|
|
|
|
|
|
+def test_generate_create_hive_iceberg_table():
|
|
|
source = json.loads(
|
|
source = json.loads(
|
|
|
'''{"sourceType": "hive", "name":"","sample":[["Bank Of America","3000000.0","US","Miami","37.6801986694",'''
|
|
'''{"sourceType": "hive", "name":"","sample":[["Bank Of America","3000000.0","US","Miami","37.6801986694",'''
|
|
|
'''"-121.92150116"],["Citi Bank","2800000.0","US","Richmond","37.5242004395","-77.4932022095"],["Deutsche Bank","2600000.0","US",'''
|
|
'''"-121.92150116"],["Citi Bank","2800000.0","US","Richmond","37.5242004395","-77.4932022095"],["Deutsche Bank","2600000.0","US",'''
|
|
@@ -1336,7 +1336,7 @@ def test_generate_create_iceberg_table():
|
|
|
'''"text","name":"Text"},{"value":"parquet","name":"Parquet"},{"value":"kudu","name":"Kudu"},{"value":"csv","name":"Csv"},'''
|
|
'''"text","name":"Text"},{"value":"parquet","name":"Parquet"},{"value":"kudu","name":"Kudu"},{"value":"csv","name":"Csv"},'''
|
|
|
'''{"value":"avro","name":"Avro"},{"value":"json","name":"Json"},{"value":"regexp","name":"Regexp"},{"value":"orc",'''
|
|
'''{"value":"avro","name":"Avro"},{"value":"json","name":"Json"},{"value":"regexp","name":"Regexp"},{"value":"orc",'''
|
|
|
'''"name":"ORC"}],"partitionColumns":[],"kuduPartitionColumns":[],"primaryKeys":[],"primaryKeyObjects":[],"importData":true,'''
|
|
'''"name":"ORC"}],"partitionColumns":[],"kuduPartitionColumns":[],"primaryKeys":[],"primaryKeyObjects":[],"importData":true,'''
|
|
|
- '''"isIceberg":true,"useCopy":false,"useDefaultLocation":true,"nonDefaultLocation":"/user/hue/data/query-hive-360.csv",'''
|
|
|
|
|
|
|
+ '''"isIceberg":true,"useCopy":false,"useDefaultLocation":false,"nonDefaultLocation":"/user/hue/data/query-hive-360.csv",'''
|
|
|
'''"hasHeader":true,"useCustomDelimiters":false,"customFieldDelimiter":",","customCollectionDelimiter":"\\\\002",'''
|
|
'''"hasHeader":true,"useCustomDelimiters":false,"customFieldDelimiter":",","customCollectionDelimiter":"\\\\002",'''
|
|
|
'''"customMapDelimiter":"\\\\003","customRegexp":""}'''
|
|
'''"customMapDelimiter":"\\\\003","customRegexp":""}'''
|
|
|
)
|
|
)
|
|
@@ -1346,7 +1346,6 @@ def test_generate_create_iceberg_table():
|
|
|
|
|
|
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
|
|
|
|
|
|
- print(sql)
|
|
|
|
|
assert '''USE default;''' in sql, sql
|
|
assert '''USE default;''' in sql, sql
|
|
|
|
|
|
|
|
statement = '''CREATE EXTERNAL TABLE IF NOT EXISTS `default`.`hue__tmp_parquet_table`
|
|
statement = '''CREATE EXTERNAL TABLE IF NOT EXISTS `default`.`hue__tmp_parquet_table`
|
|
@@ -1366,7 +1365,7 @@ TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
|
|
|
assert statement in sql, sql
|
|
assert statement in sql, sql
|
|
|
|
|
|
|
|
assert (
|
|
assert (
|
|
|
- '''CREATE TABLE `default`.`parquet_table`
|
|
|
|
|
|
|
+ '''CREATE EXTERNAL TABLE `default`.`parquet_table`
|
|
|
STORED BY ICEBERG
|
|
STORED BY ICEBERG
|
|
|
STORED AS parquet
|
|
STORED AS parquet
|
|
|
AS SELECT *
|
|
AS SELECT *
|
|
@@ -1378,6 +1377,101 @@ STORED AS parquet
|
|
|
assert '''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;''' in sql, sql
|
|
assert '''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;''' in sql, sql
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.django_db
|
|
|
|
|
+def test_generate_create_impala_iceberg_table():
|
|
|
|
|
+ source = json.loads(
|
|
|
|
|
+ '''{"sourceType": "impala", "name":"","sample":[["Bank Of America","3000000.0","US","Miami","37.6801986694",'''
|
|
|
|
|
+ '''"-121.92150116"],["Citi Bank","2800000.0","US","Richmond","37.5242004395","-77.4932022095"],["Deutsche Bank","2600000.0","US",'''
|
|
|
|
|
+ '''"Corpus Christi","40.7807998657","-73.9772033691"],["Thomson Reuters","2400000.0","US","Albany","35.7976989746",'''
|
|
|
|
|
+ '''"-78.6252975464"],'''
|
|
|
|
|
+ '''["OpenX","2200000.0","US","Des Moines","40.5411987305","-119.586898804"]],"sampleCols":[{"operations":[],"comment":"",'''
|
|
|
|
|
+ '''"nested":[],'''
|
|
|
|
|
+ '''"name":"acct_client","level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,'''
|
|
|
|
|
+ '''"partitionValue":"","multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],'''
|
|
|
|
|
+ '''"comment":"","nested":[],"name":"tran_amount","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
|
|
|
|
|
+ '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"double",'''
|
|
|
|
|
+ '''"showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"tran_country_cd","level":0,"keyType":'''
|
|
|
|
|
+ '''"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"","multiValued":false,'''
|
|
|
|
|
+ '''"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"vrfcn_city",'''
|
|
|
|
|
+ '''"level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"",'''
|
|
|
|
|
+ '''"multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],'''
|
|
|
|
|
+ '''"name":"vrfcn_city_lat","level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,'''
|
|
|
|
|
+ '''"length":100,'''
|
|
|
|
|
+ '''"partitionValue":"","multiValued":false,"unique":false,"type":"double","showProperties":false,"scale":0},{"operations":[],'''
|
|
|
|
|
+ '''"comment":"","nested":[],"name":"vrfcn_city_lon","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
|
|
|
|
|
+ '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"double","showProperties":false,'''
|
|
|
|
|
+ '''"scale":0}],"inputFormat":"file","inputFormatsAll":[{"value":"file","name":"File"},{"value":"manual","name":"Manually"},'''
|
|
|
|
|
+ '''{"value":"query","name":"SQL Query"},{"value":"table","name":"Table"}],"inputFormatsManual":[{"value":"manual","name":'''
|
|
|
|
|
+ '''"Manually"}],"inputFormats":[{"value":"file","name":"File"},{"value":"manual","name":"Manually"},{"value":"query","name":'''
|
|
|
|
|
+ '''"SQL Query"},{"value":"table","name":"Table"}],"path":"/user/hue/data/query-impala-360.csv","isObjectStore":false,"table":"",'''
|
|
|
|
|
+ '''"tableName":"","databaseName":"default","apiHelperType":"impala","query":"","draggedQuery":"","format":{"type":"csv",'''
|
|
|
|
|
+ '''"fieldSeparator":",","recordSeparator":"\\n","quoteChar":"\\"","hasHeader":true,"status":0},"show":true,"defaultName":'''
|
|
|
|
|
+ '''"default.query-impala-360"}'''
|
|
|
|
|
+ )
|
|
|
|
|
+ destination = json.loads(
|
|
|
|
|
+ '''{"isTransactional": false, "isInsertOnly": false, "sourceType": "impala", "name":"default.parquet_table"'''
|
|
|
|
|
+ ''',"apiHelperType":"impala","description":"","outputFormat":"table","outputFormatsList":[{"name":"Table","value":"table"},'''
|
|
|
|
|
+ '''{"name":"Solr index","value":"index"},{"name":"File","value":"file"},{"name":"Database","value":"database"}],'''
|
|
|
|
|
+ '''"outputFormats":[{"name":"Table","value":"table"},{"name":"Solr index","value":"index"}],"columns":[{"operations":[],'''
|
|
|
|
|
+ '''"comment":"","nested":[],"name":"acct_client","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
|
|
|
|
|
+ '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"string","showProperties":'''
|
|
|
|
|
+ '''false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"tran_amount","level":0,"keyType":"string","required":false,'''
|
|
|
|
|
+ '''"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":'''
|
|
|
|
|
+ '''"double","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"tran_country_cd","level":0,'''
|
|
|
|
|
+ '''"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"",'''
|
|
|
|
|
+ '''"multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],"comment":"","nested":'''
|
|
|
|
|
+ '''[],"name":"vrfcn_city","level":0,"keyType":"string","required":false,"precision":10,"keep":true,"isPartition":false,"length":'''
|
|
|
|
|
+ '''100,"partitionValue":"","multiValued":false,"unique":false,"type":"string","showProperties":false,"scale":0},{"operations":[],'''
|
|
|
|
|
+ '''"comment":"","nested":[],"name":"vrfcn_city_lat","level":0,"keyType":"string","required":false,"precision":10,"keep":true,'''
|
|
|
|
|
+ '''"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,"type":"double","showProperties":'''
|
|
|
|
|
+ '''false,"scale":0},{"operations":[],"comment":"","nested":[],"name":"vrfcn_city_lon","level":0,"keyType":"string","required":'''
|
|
|
|
|
+ '''false,"precision":10,"keep":true,"isPartition":false,"length":100,"partitionValue":"","multiValued":false,"unique":false,'''
|
|
|
|
|
+ '''"type":"double","showProperties":false,"scale":0}],"bulkColumnNames":"acct_client,tran_amount,tran_country_cd,vrfcn_city,'''
|
|
|
|
|
+ '''vrfcn_city_lat,vrfcn_city_lon","showProperties":false,"isTargetExisting":false,"isTargetChecking":false,"existingTargetUrl":'''
|
|
|
|
|
+ '''"","tableName":"parquet_table","databaseName":"default","tableFormat":"parquet","KUDU_DEFAULT_RANGE_PARTITION_COLUMN":'''
|
|
|
|
|
+ '''{"values":[{"value":""}],"name":"VALUES","lower_val":0,"include_lower_val":"<=","upper_val":1,"include_upper_val":"<="},'''
|
|
|
|
|
+ '''"KUDU_DEFAULT_PARTITION_COLUMN":{"columns":[],"range_partitions":[{"values":[{"value":""}],"name":"VALUES","lower_val":0,'''
|
|
|
|
|
+ '''"include_lower_val":"<=","upper_val":1,"include_upper_val":"<="}],"name":"HASH","int_val":16},"tableFormats":[{"value":'''
|
|
|
|
|
+ '''"text","name":"Text"},{"value":"parquet","name":"Parquet"},{"value":"kudu","name":"Kudu"},{"value":"csv","name":"Csv"},'''
|
|
|
|
|
+ '''{"value":"avro","name":"Avro"},{"value":"json","name":"Json"},{"value":"regexp","name":"Regexp"},{"value":"orc",'''
|
|
|
|
|
+ '''"name":"ORC"}],"partitionColumns":[],"kuduPartitionColumns":[],"primaryKeys":[],"primaryKeyObjects":[],"importData":true,'''
|
|
|
|
|
+ '''"isIceberg":true,"useCopy":false,"useDefaultLocation":false,"nonDefaultLocation":"/user/hue/data/query-impala-360.csv",'''
|
|
|
|
|
+ '''"hasHeader":true,"useCustomDelimiters":false,"customFieldDelimiter":",","customCollectionDelimiter":"\\\\002",'''
|
|
|
|
|
+ '''"customMapDelimiter":"\\\\003","customRegexp":""}'''
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ path = {'isDir': False, 'split': ('/user/hue/data', 'query-impala-360.csv'), 'listdir': ['/user/hue/data']}
|
|
|
|
|
+ request = MockRequest(fs=MockFs(path=path))
|
|
|
|
|
+
|
|
|
|
|
+ sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
|
|
|
+
|
|
|
|
|
+ assert '''USE default;''' in sql, sql
|
|
|
|
|
+
|
|
|
|
|
+ statement = '''CREATE EXTERNAL TABLE IF NOT EXISTS `default`.`hue__tmp_parquet_table`
|
|
|
|
|
+(
|
|
|
|
|
+ `acct_client` string ,
|
|
|
|
|
+ `tran_amount` double ,
|
|
|
|
|
+ `tran_country_cd` string ,
|
|
|
|
|
+ `vrfcn_city` string ,
|
|
|
|
|
+ `vrfcn_city_lat` double ,
|
|
|
|
|
+ `vrfcn_city_lon` double ) ROW FORMAT DELIMITED
|
|
|
|
|
+ FIELDS TERMINATED BY ','
|
|
|
|
|
+ STORED AS TextFile LOCATION '/user/hue/data'
|
|
|
|
|
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
|
|
|
|
|
+;'''
|
|
|
|
|
+ assert statement in sql, sql
|
|
|
|
|
+
|
|
|
|
|
+ assert (
|
|
|
|
|
+ '''CREATE TABLE `default`.`parquet_table`
|
|
|
|
|
+ STORED AS ICEBERG
|
|
|
|
|
+ AS SELECT *
|
|
|
|
|
+ FROM `default`.`hue__tmp_parquet_table`;'''
|
|
|
|
|
+ in sql
|
|
|
|
|
+ ), sql
|
|
|
|
|
+
|
|
|
|
|
+ assert '''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;''' in sql, sql
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@pytest.mark.django_db
|
|
@pytest.mark.django_db
|
|
|
def test_generate_create_orc_table_transactional():
|
|
def test_generate_create_orc_table_transactional():
|
|
|
source = json.loads(
|
|
source = json.loads(
|