Przeglądaj źródła

HUE-9150 [importer] Simplify SQL statement test comparison

Sort table properties.
Romain 5 lat temu
rodzic
commit
0d20d65532

+ 3 - 1
desktop/libs/indexer/src/indexer/indexers/sql.py

@@ -171,7 +171,9 @@ class SQLIndexer(object):
     tbl_properties = OrderedDict()
     tbl_properties = OrderedDict()
     if skip_header:
     if skip_header:
       tbl_properties['skip.header.line.count'] = '1'
       tbl_properties['skip.header.line.count'] = '1'
-    tbl_properties['transactional'] = 'false' # The temp table is not transactional, but final table can be if is_transactional. tbl_properties that don't exist in previous versions can safely be added without error
+    # The temp table is not transactional, but final table can be if is_transactional.
+    # tbl_properties that don't exist in previous versions can safely be added without error.
+    tbl_properties['transactional'] = 'false'
 
 
     sql += django_mako.render_to_string("gen/create_table_statement.mako", {
     sql += django_mako.render_to_string("gen/create_table_statement.mako", {
         'table': {
         'table': {

+ 22 - 29
desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -20,7 +20,6 @@ from builtins import object
 import json
 import json
 import sys
 import sys
 
 
-from collections import OrderedDict
 from nose.tools import assert_equal, assert_true
 from nose.tools import assert_equal, assert_true
 
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
@@ -28,18 +27,13 @@ from useradmin.models import User
 
 
 from indexer.indexers.sql import SQLIndexer
 from indexer.indexers.sql import SQLIndexer
 
 
+
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
   from unittest.mock import patch, Mock, MagicMock
   from unittest.mock import patch, Mock, MagicMock
 else:
 else:
   from mock import patch, Mock, MagicMock
   from mock import patch, Mock, MagicMock
 
 
 
 
-TABLE_PROPERTIES = [
-  '%s = %s' % (key, val)
-  for key, val in OrderedDict({"transactional": "false", "skip.header.line.count": "1"}).items()
-]
-
-
 class TestSQLIndexer(object):
 class TestSQLIndexer(object):
 
 
   def setUp(self):
   def setUp(self):
@@ -145,7 +139,7 @@ def test_generate_create_text_table_with_data_partition():
 
 
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
 
 
-  assert_true('''USE default;''' in  sql, sql)
+  assert_true('''USE default;''' in sql, sql)
 
 
   statement = '''CREATE TABLE `default`.`customer_stats`
   statement = '''CREATE TABLE `default`.`customer_stats`
 (
 (
@@ -159,11 +153,11 @@ ROW FORMAT   DELIMITED
     FIELDS TERMINATED BY ','
     FIELDS TERMINATED BY ','
     COLLECTION ITEMS TERMINATED BY '\\002'
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
     MAP KEYS TERMINATED BY '\\003'
-  STORED AS TextFile TBLPROPERTIES(%s)
-;''' % TABLE_PROPERTIES
+  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+;'''
   assert_true(statement in sql, sql)
   assert_true(statement in sql, sql)
 
 
-  assert_true('''LOAD DATA INPATH '/user/romain/customer_stats.csv' INTO TABLE `default`.`customer_stats` PARTITION (new_field_1='AAA');''' in  sql, sql)
+  assert_true('''LOAD DATA INPATH '/user/romain/customer_stats.csv' INTO TABLE `default`.`customer_stats` PARTITION (new_field_1='AAA');''' in sql, sql)
 
 
 
 
 def test_generate_create_kudu_table_with_data():
 def test_generate_create_kudu_table_with_data():
@@ -173,7 +167,7 @@ def test_generate_create_kudu_table_with_data():
 
 
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
 
 
-  assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_index_data`;''' in  sql, sql)
+  assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_index_data`;''' in sql, sql)
 
 
   statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_index_data`
   statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_index_data`
 (
 (
@@ -198,8 +192,8 @@ def test_generate_create_kudu_table_with_data():
 ROW FORMAT   DELIMITED
 ROW FORMAT   DELIMITED
     FIELDS TERMINATED BY ','
     FIELDS TERMINATED BY ','
   STORED AS TextFile LOCATION '/A'
   STORED AS TextFile LOCATION '/A'
-TBLPROPERTIES(%s)''' % TABLE_PROPERTIES
-  assert_true(statement in sql in sql, sql)
+TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")'''
+  assert_true(statement in sql, sql)
 
 
   assert_true('''CREATE TABLE `default`.`index_data` COMMENT "Big Data"
   assert_true('''CREATE TABLE `default`.`index_data` COMMENT "Big Data"
         PRIMARY KEY (id)
         PRIMARY KEY (id)
@@ -209,7 +203,7 @@ TBLPROPERTIES(%s)''' % TABLE_PROPERTIES
         'kudu.num_tablet_replicas' = '1'
         'kudu.num_tablet_replicas' = '1'
         )
         )
         AS SELECT `id`, `business_id`, `date`, `funny`, `stars`, `text`, `type`, `useful`, `user_id`, `name`, `full_address`, `latitude`, `longitude`, `neighborhoods`, `open`, `review_count`, `state`
         AS SELECT `id`, `business_id`, `date`, `funny`, `stars`, `text`, `type`, `useful`, `user_id`, `name`, `full_address`, `latitude`, `longitude`, `neighborhoods`, `open`, `review_count`, `state`
-        FROM `default`.`hue__tmp_index_data`;''' in  sql, sql)
+        FROM `default`.`hue__tmp_index_data`;''' in sql, sql)
 
 
 
 
 def test_generate_create_parquet_table():
 def test_generate_create_parquet_table():
@@ -221,7 +215,7 @@ def test_generate_create_parquet_table():
 
 
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
 
 
-  assert_true('''USE default;''' in  sql, sql)
+  assert_true('''USE default;''' in sql, sql)
 
 
   statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
   statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
 (
 (
@@ -235,18 +229,17 @@ def test_generate_create_parquet_table():
     COLLECTION ITEMS TERMINATED BY '\\002'
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
     MAP KEYS TERMINATED BY '\\003'
   STORED AS TextFile LOCATION '/user/hue/data'
   STORED AS TextFile LOCATION '/user/hue/data'
-TBLPROPERTIES(%s)
-;''' % TABLE_PROPERTIES
-  assert_true(statement in  sql, sql)
+TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+;'''
+  assert_true(statement in sql, sql)
 
 
   assert_true('''CREATE TABLE `default`.`parquet_table`
   assert_true('''CREATE TABLE `default`.`parquet_table`
         STORED AS parquet
         STORED AS parquet
         AS SELECT *
         AS SELECT *
         FROM `default`.`hue__tmp_parquet_table`;
         FROM `default`.`hue__tmp_parquet_table`;
-''' in  sql, sql)
+''' in sql, sql)
 
 
-  assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;
-''' in  sql, sql)
+  assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;''' in sql, sql)
 
 
 
 
 def test_generate_create_orc_table_transactional():
 def test_generate_create_orc_table_transactional():
@@ -258,7 +251,7 @@ def test_generate_create_orc_table_transactional():
 
 
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
   sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(source, destination).get_str()
 
 
-  assert_true('''USE default;''' in  sql, sql)
+  assert_true('''USE default;''' in sql, sql)
 
 
   statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
   statement = '''CREATE EXTERNAL TABLE `default`.`hue__tmp_parquet_table`
 (
 (
@@ -272,19 +265,19 @@ def test_generate_create_orc_table_transactional():
     COLLECTION ITEMS TERMINATED BY '\\002'
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
     MAP KEYS TERMINATED BY '\\003'
   STORED AS TextFile LOCATION '/user/hue/data'
   STORED AS TextFile LOCATION '/user/hue/data'
-TBLPROPERTIES(%s)
-;''' % TABLE_PROPERTIES
-  assert_true(statement in sql in  sql, sql)
+TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+;'''
+  assert_true(statement in sql, sql)
 
 
   assert_true('''CREATE TABLE `default`.`parquet_table`
   assert_true('''CREATE TABLE `default`.`parquet_table`
         STORED AS orc
         STORED AS orc
 TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
 TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
         AS SELECT *
         AS SELECT *
         FROM `default`.`hue__tmp_parquet_table`;
         FROM `default`.`hue__tmp_parquet_table`;
-''' in  sql, sql)
+''' in sql, sql)
 
 
   assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;
   assert_true('''DROP TABLE IF EXISTS `default`.`hue__tmp_parquet_table`;
-''' in  sql, sql)
+''' in sql, sql)
 
 
 
 
 def test_generate_create_empty_kudu_table():
 def test_generate_create_empty_kudu_table():
@@ -305,4 +298,4 @@ def test_generate_create_empty_kudu_table():
   `vrfcn_city_lat` double ,
   `vrfcn_city_lat` double ,
   `vrfcn_city_lon` double , PRIMARY KEY (acct_client)
   `vrfcn_city_lon` double , PRIMARY KEY (acct_client)
 )   STORED AS kudu TBLPROPERTIES("transactional" = "false")
 )   STORED AS kudu TBLPROPERTIES("transactional" = "false")
-;''' in  sql, sql)
+;''' in sql, sql)