|
@@ -20,7 +20,6 @@ from builtins import object
|
|
|
import json
|
|
import json
|
|
|
import sys
|
|
import sys
|
|
|
|
|
|
|
|
-from collections import OrderedDict
|
|
|
|
|
from nose.tools import assert_equal, assert_true
|
|
from nose.tools import assert_equal, assert_true
|
|
|
|
|
|
|
|
from desktop.lib.django_test_util import make_logged_in_client
|
|
from desktop.lib.django_test_util import make_logged_in_client
|
|
@@ -28,18 +27,13 @@ from useradmin.models import User
|
|
|
|
|
|
|
|
from indexer.indexers.sql import SQLIndexer
|
|
from indexer.indexers.sql import SQLIndexer
|
|
|
|
|
|
|
|
|
|
+
|
|
|
if sys.version_info[0] > 2:
|
|
if sys.version_info[0] > 2:
|
|
|
from unittest.mock import patch, Mock, MagicMock
|
|
from unittest.mock import patch, Mock, MagicMock
|
|
|
else:
|
|
else:
|
|
|
from mock import patch, Mock, MagicMock
|
|
from mock import patch, Mock, MagicMock
|
|
|
|
|
|
|
|
|
|
|
|
|
-TABLE_PROPERTIES = [
|
|
|
|
|
- '%s = %s' % (key, val)
|
|
|
|
|
- for key, val in OrderedDict({"transactional": "false", "skip.header.line.count": "1"}).items()
|
|
|
|
|
-]
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
class TestSQLIndexer(object):
|
|
class TestSQLIndexer(object):
|
|
|
|
|
|
|
|
def setUp(self):
|
|
def setUp(self):
|
|
@@ -145,7 +139,7 @@ def test_generate_create_text_table_with_data_partition():
|
|
|
|
|
|
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
|
|
|
|
|
|
- assert_true('''USE default;''' in sql, sql)
|
|
|
|
|
|
|
+ assert_true('''USE default;''' in sql, sql)
|
|
|
|
|
|
|
|
statement = '''CREATE TABLE `default`.`customer_stats`
|
|
statement = '''CREATE TABLE `default`.`customer_stats`
|
|
|
(
|
|
(
|
|
@@ -159,11 +153,11 @@ ROW FORMAT DELIMITED
|
|
|
FIELDS TERMINATED BY ','
|
|
FIELDS TERMINATED BY ','
|
|
|
COLLECTION ITEMS TERMINATED BY '\\002'
|
|
COLLECTION ITEMS TERMINATED BY '\\002'
|
|
|
MAP KEYS TERMINATED BY '\\003'
|
|
MAP KEYS TERMINATED BY '\\003'
|
|
|
- STORED AS TextFile TBLPROPERTIES(%s)
|
|
|
|
|
-;''' % TABLE_PROPERTIES
|
|
|
|
|
|
|
+ STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
|
|
|
|
|
+;'''
|
|
|
assert_true(statement in sql, sql)
|
|
assert_true(statement in sql, sql)
|
|
|
|
|
|
|
|
- assert_true('''LOAD DATA INPATH '/user/romain/customer_stats.csv' INTO TABLE `default`.`customer_stats` PARTITION (new_field_1='AAA');''' in sql, sql)
|
|
|
|
|
|
|
+ assert_true('''LOAD DATA INPATH '/user/romain/customer_stats.csv' INTO TABLE `default`.`customer_stats` PARTITION (new_field_1='AAA');''' in sql, sql)
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_generate_create_kudu_table_with_data():
|
|
def test_generate_create_kudu_table_with_data():
|
|
@@ -173,7 +167,7 @@ def test_generate_create_kudu_table_with_data():
|
|
|
|
|
|
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
|
|
|
|
|
|
- assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_index_data`;''' in sql, sql)
|
|
|
|
|
|
|
+ assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_index_data`;''' in sql, sql)
|
|
|
|
|
|
|
|
statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_index_data`
|
|
statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_index_data`
|
|
|
(
|
|
(
|
|
@@ -198,8 +192,8 @@ def test_generate_create_kudu_table_with_data():
|
|
|
ROW FORMAT DELIMITED
|
|
ROW FORMAT DELIMITED
|
|
|
FIELDS TERMINATED BY ','
|
|
FIELDS TERMINATED BY ','
|
|
|
STORED AS TextFile LOCATION '/A'
|
|
STORED AS TextFile LOCATION '/A'
|
|
|
-TBLPROPERTIES(%s)''' % TABLE_PROPERTIES
|
|
|
|
|
- assert_true(statement in sql in sql, sql)
|
|
|
|
|
|
|
+TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")'''
|
|
|
|
|
+ assert_true(statement in sql, sql)
|
|
|
|
|
|
|
|
assert_true('''CREATE TABLE `default`.`index_data` COMMENT "Big Data"
|
|
assert_true('''CREATE TABLE `default`.`index_data` COMMENT "Big Data"
|
|
|
PRIMARY KEY (id)
|
|
PRIMARY KEY (id)
|
|
@@ -209,7 +203,7 @@ TBLPROPERTIES(%s)''' % TABLE_PROPERTIES
|
|
|
'kudu.num_tablet_replicas' = '1'
|
|
'kudu.num_tablet_replicas' = '1'
|
|
|
)
|
|
)
|
|
|
AS SELECT `id`, `business_id`, `date`, `funny`, `stars`, `text`, `type`, `useful`, `user_id`, `name`, `full_address`, `latitude`, `longitude`, `neighborhoods`, `open`, `review_count`, `state`
|
|
AS SELECT `id`, `business_id`, `date`, `funny`, `stars`, `text`, `type`, `useful`, `user_id`, `name`, `full_address`, `latitude`, `longitude`, `neighborhoods`, `open`, `review_count`, `state`
|
|
|
- FROM `default`.`hue__tmp_index_data`;''' in sql, sql)
|
|
|
|
|
|
|
+ FROM `default`.`hue__tmp_index_data`;''' in sql, sql)
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_generate_create_parquet_table():
|
|
def test_generate_create_parquet_table():
|
|
@@ -221,7 +215,7 @@ def test_generate_create_parquet_table():
|
|
|
|
|
|
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
|
|
|
|
|
|
- assert_true('''USE default;''' in sql, sql)
|
|
|
|
|
|
|
+ assert_true('''USE default;''' in sql, sql)
|
|
|
|
|
|
|
|
statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
|
|
statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
|
|
|
(
|
|
(
|
|
@@ -235,18 +229,17 @@ def test_generate_create_parquet_table():
|
|
|
COLLECTION ITEMS TERMINATED BY '\\002'
|
|
COLLECTION ITEMS TERMINATED BY '\\002'
|
|
|
MAP KEYS TERMINATED BY '\\003'
|
|
MAP KEYS TERMINATED BY '\\003'
|
|
|
STORED AS TextFile LOCATION '/user/hue/data'
|
|
STORED AS TextFile LOCATION '/user/hue/data'
|
|
|
-TBLPROPERTIES(%s)
|
|
|
|
|
-;''' % TABLE_PROPERTIES
|
|
|
|
|
- assert_true(statement in sql, sql)
|
|
|
|
|
|
|
+TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
|
|
|
|
|
+;'''
|
|
|
|
|
+ assert_true(statement in sql, sql)
|
|
|
|
|
|
|
|
assert_true('''CREATE TABLE `default`.`parquet_table`
|
|
assert_true('''CREATE TABLE `default`.`parquet_table`
|
|
|
STORED AS parquet
|
|
STORED AS parquet
|
|
|
AS SELECT *
|
|
AS SELECT *
|
|
|
FROM `default`.`hue__tmp_parquet_table`;
|
|
FROM `default`.`hue__tmp_parquet_table`;
|
|
|
-''' in sql, sql)
|
|
|
|
|
|
|
+''' in sql, sql)
|
|
|
|
|
|
|
|
- assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;
|
|
|
|
|
-''' in sql, sql)
|
|
|
|
|
|
|
+ assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;''' in sql, sql)
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_generate_create_orc_table_transactional():
|
|
def test_generate_create_orc_table_transactional():
|
|
@@ -258,7 +251,7 @@ def test_generate_create_orc_table_transactional():
|
|
|
|
|
|
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
|
|
|
|
|
|
|
|
- assert_true('''USE default;''' in sql, sql)
|
|
|
|
|
|
|
+ assert_true('''USE default;''' in sql, sql)
|
|
|
|
|
|
|
|
statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
|
|
statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
|
|
|
(
|
|
(
|
|
@@ -272,19 +265,19 @@ def test_generate_create_orc_table_transactional():
|
|
|
COLLECTION ITEMS TERMINATED BY '\\002'
|
|
COLLECTION ITEMS TERMINATED BY '\\002'
|
|
|
MAP KEYS TERMINATED BY '\\003'
|
|
MAP KEYS TERMINATED BY '\\003'
|
|
|
STORED AS TextFile LOCATION '/user/hue/data'
|
|
STORED AS TextFile LOCATION '/user/hue/data'
|
|
|
-TBLPROPERTIES(%s)
|
|
|
|
|
-;''' % TABLE_PROPERTIES
|
|
|
|
|
- assert_true(statement in sql in sql, sql)
|
|
|
|
|
|
|
+TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
|
|
|
|
|
+;'''
|
|
|
|
|
+ assert_true(statement in sql, sql)
|
|
|
|
|
|
|
|
assert_true('''CREATE TABLE `default`.`parquet_table`
|
|
assert_true('''CREATE TABLE `default`.`parquet_table`
|
|
|
STORED AS orc
|
|
STORED AS orc
|
|
|
TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
|
|
TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
|
|
|
AS SELECT *
|
|
AS SELECT *
|
|
|
FROM `default`.`hue__tmp_parquet_table`;
|
|
FROM `default`.`hue__tmp_parquet_table`;
|
|
|
-''' in sql, sql)
|
|
|
|
|
|
|
+''' in sql, sql)
|
|
|
|
|
|
|
|
assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;
|
|
assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;
|
|
|
-''' in sql, sql)
|
|
|
|
|
|
|
+''' in sql, sql)
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_generate_create_empty_kudu_table():
|
|
def test_generate_create_empty_kudu_table():
|
|
@@ -305,4 +298,4 @@ def test_generate_create_empty_kudu_table():
|
|
|
`vrfcn_city_lat` double ,
|
|
`vrfcn_city_lat` double ,
|
|
|
`vrfcn_city_lon` double , PRIMARY KEY (acct_client)
|
|
`vrfcn_city_lon` double , PRIMARY KEY (acct_client)
|
|
|
) STORED AS kudu TBLPROPERTIES("transactional" = "false")
|
|
) STORED AS kudu TBLPROPERTIES("transactional" = "false")
|
|
|
-;''' in sql, sql)
|
|
|
|
|
|
|
+;''' in sql, sql)
|