Selaa lähdekoodia

[indexer] Switch to single quotes for TBLPROPERTIES values

The new Hive does not accept those currently.
Romain Rigaux 4 vuotta sitten
vanhempi
commit
b80a403b67

+ 28 - 33
desktop/libs/indexer/src/indexer/indexers/sql.py

@@ -236,7 +236,7 @@ class SQLIndexer(object):
         PARTITION BY HASH PARTITIONS 16
         PARTITION BY HASH PARTITIONS 16
         STORED AS %(file_format)s
         STORED AS %(file_format)s
         TBLPROPERTIES(
         TBLPROPERTIES(
-        'kudu.num_tablet_replicas' = '1'
+        'kudu.num_tablet_replicas'='1'
         )""" % {
         )""" % {
           'file_format': file_format,
           'file_format': file_format,
           'primary_keys': ', '.join(primary_keys)
           'primary_keys': ', '.join(primary_keys)
@@ -245,7 +245,7 @@ class SQLIndexer(object):
         columns_list = ['*']
         columns_list = ['*']
         extra_create_properties = 'STORED AS %(file_format)s' % {'file_format': file_format}
         extra_create_properties = 'STORED AS %(file_format)s' % {'file_format': file_format}
         if is_transactional:
         if is_transactional:
-          extra_create_properties += '\nTBLPROPERTIES("transactional"="true", "transactional_properties"="%s")' % \
+          extra_create_properties += "\nTBLPROPERTIES('transactional'='true', 'transactional_properties'='%s')" % \
               default_transactional_type
               default_transactional_type
 
 
       sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`%(comment)s
       sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`%(comment)s
@@ -320,12 +320,11 @@ class SQLIndexer(object):
             col['type'] = 'VARCHAR(255)'
             col['type'] = 'VARCHAR(255)'
 
 
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
-%(columns)s);
-      ''' % {
-              'database': database,
-              'table_name': table_name,
-              'columns': ',\n'.join(['  `%(name)s` %(type)s' % col for col in columns]),
-            }
+%(columns)s);\n''' % {
+        'database': database,
+        'table_name': table_name,
+        'columns': ',\n'.join(['  `%(name)s` %(type)s' % col for col in columns]),
+      }
 
 
     elif dialect == 'phoenix':
     elif dialect == 'phoenix':
 
 
@@ -335,8 +334,7 @@ class SQLIndexer(object):
 
 
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
 %(columns)s
 %(columns)s
-CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
-''' % {
+CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));\n''' % {
           'database': database,
           'database': database,
           'table_name': table_name,
           'table_name': table_name,
           'columns': ',\n'.join(['  %(name)s %(type)s' % col for col in columns]),
           'columns': ',\n'.join(['  %(name)s %(type)s' % col for col in columns]),
@@ -345,12 +343,11 @@ CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
 
 
     elif dialect == 'impala':
     elif dialect == 'impala':
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s_tmp (
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s_tmp (
-%(columns)s);
-      ''' % {
-                'database': database,
-                'table_name': table_name,
-                'columns': ',\n'.join(['  `%(name)s` string' % col for col in columns]),
-            }                                                 # Impala does not implicitly cast between string and numeric or Boolean types.
+%(columns)s);\n''' % {
+          'database': database,
+          'table_name': table_name,
+          'columns': ',\n'.join(['  `%(name)s` string' % col for col in columns]),
+      }                                                 # Impala does not implicitly cast between string and numeric or Boolean types.
 
 
     path = urllib_unquote(source['path'])
     path = urllib_unquote(source['path'])
 
 
@@ -370,23 +367,21 @@ CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
           csv_rows = str(_csv_rows)[1:-1]
           csv_rows = str(_csv_rows)[1:-1]
 
 
           if dialect in ('hive', 'mysql'):
           if dialect in ('hive', 'mysql'):
-            sql += '''\nINSERT INTO %(database)s.%(table_name)s VALUES %(csv_rows)s;
-            '''% {
-                    'database': database,
-                    'table_name': table_name,
-                    'csv_rows': csv_rows
-                  }
+            sql += '''\nINSERT INTO %(database)s.%(table_name)s VALUES %(csv_rows)s;\n'''% {
+              'database': database,
+              'table_name': table_name,
+              'csv_rows': csv_rows
+            }
           elif dialect == 'phoenix':
           elif dialect == 'phoenix':
             for csv_row in _csv_rows:
             for csv_row in _csv_rows:
               _sql = ', '.join([ "'{0}'".format(col_val) if columns[count]['type'] in ('CHAR(255)', 'timestamp') \
               _sql = ', '.join([ "'{0}'".format(col_val) if columns[count]['type'] in ('CHAR(255)', 'timestamp') \
                 else '{0}'.format(col_val) for count, col_val in enumerate(csv_row)])
                 else '{0}'.format(col_val) for count, col_val in enumerate(csv_row)])
 
 
-              sql += '''\nUPSERT INTO %(database)s.%(table_name)s VALUES (%(csv_row)s);
-            ''' % {
-                    'database': database,
-                    'table_name': table_name,
-                    'csv_row': _sql
-                  }
+              sql += '''\nUPSERT INTO %(database)s.%(table_name)s VALUES (%(csv_row)s);\n''' % {
+                'database': database,
+                'table_name': table_name,
+                'csv_row': _sql
+              }
           elif dialect == 'impala':
           elif dialect == 'impala':
              # casting from string to boolean is not allowed in impala so string -> int -> bool
              # casting from string to boolean is not allowed in impala so string -> int -> bool
             sql_ = ',\n'.join([
             sql_ = ',\n'.join([
@@ -396,11 +391,11 @@ CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
 
 
             sql += '''\nINSERT INTO %(database)s.%(table_name)s_tmp VALUES %(csv_rows)s;\n\nCREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s
             sql += '''\nINSERT INTO %(database)s.%(table_name)s_tmp VALUES %(csv_rows)s;\n\nCREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s
 AS SELECT\n%(sql_)s\nFROM  %(database)s.%(table_name)s_tmp;\n\nDROP TABLE IF EXISTS %(database)s.%(table_name)s_tmp;'''% {
 AS SELECT\n%(sql_)s\nFROM  %(database)s.%(table_name)s_tmp;\n\nDROP TABLE IF EXISTS %(database)s.%(table_name)s_tmp;'''% {
-                    'database': database,
-                    'table_name': table_name,
-                    'csv_rows': csv_rows,
-                    'sql_': sql_
-                  }
+              'database': database,
+              'table_name': table_name,
+              'csv_rows': csv_rows,
+              'sql_': sql_
+            }
 
 
     on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \
     on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \
         '?source_type=' + source_type
         '?source_type=' + source_type

+ 22 - 22
desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -89,14 +89,14 @@ ROW FORMAT   SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
     "quoteChar"     = """,
     "quoteChar"     = """,
     "escapeChar"    = "\\\\"
     "escapeChar"    = "\\\\"
     )
     )
-  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+  STORED AS TextFile TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;
 ;
 
 
 LOAD DATA INPATH 'hdfs:///path/data.csv' INTO TABLE `default`.`hue__tmp_export_table` PARTITION (day='20200101');
 LOAD DATA INPATH 'hdfs:///path/data.csv' INTO TABLE `default`.`hue__tmp_export_table` PARTITION (day='20200101');
 
 
 CREATE TABLE `default`.`export_table` COMMENT "No comment!"
 CREATE TABLE `default`.`export_table` COMMENT "No comment!"
         STORED AS csv
         STORED AS csv
-TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
+TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only')
         AS SELECT *
         AS SELECT *
         FROM `default`.`hue__tmp_export_table`;
         FROM `default`.`hue__tmp_export_table`;
 
 
@@ -216,7 +216,7 @@ ROW FORMAT   DELIMITED
     FIELDS TERMINATED BY ','
     FIELDS TERMINATED BY ','
     COLLECTION ITEMS TERMINATED BY '\\002'
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
     MAP KEYS TERMINATED BY '\\003'
-  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+  STORED AS TextFile TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
 ;'''
   assert_true(statement in sql, sql)
   assert_true(statement in sql, sql)
 
 
@@ -334,7 +334,7 @@ def test_generate_create_kudu_table_with_data():
 ROW FORMAT   DELIMITED
 ROW FORMAT   DELIMITED
     FIELDS TERMINATED BY ','
     FIELDS TERMINATED BY ','
   STORED AS TextFile LOCATION '/A'
   STORED AS TextFile LOCATION '/A'
-TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")'''
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')'''
     assert_true(statement in sql, sql)
     assert_true(statement in sql, sql)
 
 
     assert_true('''CREATE TABLE `default`.`index_data` COMMENT "Big Data"
     assert_true('''CREATE TABLE `default`.`index_data` COMMENT "Big Data"
@@ -342,7 +342,7 @@ TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")'''
         PARTITION BY HASH PARTITIONS 16
         PARTITION BY HASH PARTITIONS 16
         STORED AS kudu
         STORED AS kudu
         TBLPROPERTIES(
         TBLPROPERTIES(
-        'kudu.num_tablet_replicas' = '1'
+        'kudu.num_tablet_replicas'='1'
         )
         )
         AS SELECT `id`, `business_id`, `date`, `funny`, `stars`, `text`, `type`, `useful`, `user_id`, `name`, '''
         AS SELECT `id`, `business_id`, `date`, `funny`, `stars`, `text`, `type`, `useful`, `user_id`, `name`, '''
         '''`full_address`, `latitude`, `longitude`, `neighborhoods`, `open`, `review_count`, `state`
         '''`full_address`, `latitude`, `longitude`, `neighborhoods`, `open`, `review_count`, `state`
@@ -429,7 +429,7 @@ def test_generate_create_parquet_table():
     COLLECTION ITEMS TERMINATED BY '\\002'
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
     MAP KEYS TERMINATED BY '\\003'
   STORED AS TextFile LOCATION '/user/hue/data'
   STORED AS TextFile LOCATION '/user/hue/data'
-TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
 ;'''
   assert_true(statement in sql, sql)
   assert_true(statement in sql, sql)
 
 
@@ -521,13 +521,13 @@ def test_generate_create_orc_table_transactional():
     COLLECTION ITEMS TERMINATED BY '\\002'
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
     MAP KEYS TERMINATED BY '\\003'
   STORED AS TextFile LOCATION '/user/hue/data'
   STORED AS TextFile LOCATION '/user/hue/data'
-TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
 ;'''
   assert_true(statement in sql, sql)
   assert_true(statement in sql, sql)
 
 
   assert_true('''CREATE TABLE `default`.`parquet_table`
   assert_true('''CREATE TABLE `default`.`parquet_table`
         STORED AS orc
         STORED AS orc
-TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
+TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only')
         AS SELECT *
         AS SELECT *
         FROM `default`.`hue__tmp_parquet_table`;
         FROM `default`.`hue__tmp_parquet_table`;
 ''' in sql, sql)
 ''' in sql, sql)
@@ -580,7 +580,7 @@ def test_generate_create_empty_kudu_table():
   `vrfcn_city` string ,
   `vrfcn_city` string ,
   `vrfcn_city_lat` double ,
   `vrfcn_city_lat` double ,
   `vrfcn_city_lon` double , PRIMARY KEY (acct_client)
   `vrfcn_city_lon` double , PRIMARY KEY (acct_client)
-)   STORED AS kudu TBLPROPERTIES("transactional" = "false")
+)   STORED AS kudu TBLPROPERTIES('transactional'='false')
 ;''' in sql, sql)
 ;''' in sql, sql)
 
 
 
 
@@ -742,7 +742,7 @@ def test_create_ddl_with_nonascii():
     FIELDS TERMINATED BY ','
     FIELDS TERMINATED BY ','
     COLLECTION ITEMS TERMINATED BY '\\002'
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
     MAP KEYS TERMINATED BY '\\003'
-  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+  STORED AS TextFile TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
 ;'''
   assert_true(statement in sql, sql)
   assert_true(statement in sql, sql)
 
 
@@ -752,7 +752,7 @@ def test_create_ddl_with_nonascii():
 
 
   statement = '''CREATE TABLE `default`.`renamed_chinese_cities_gb2312`
   statement = '''CREATE TABLE `default`.`renamed_chinese_cities_gb2312`
         STORED AS TextFile
         STORED AS TextFile
-TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
+TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only')
         AS SELECT *
         AS SELECT *
         FROM `default`.`hue__tmp_renamed_chinese_cities_gb2312`;'''
         FROM `default`.`hue__tmp_renamed_chinese_cities_gb2312`;'''
   assert_true(statement in sql, sql)
   assert_true(statement in sql, sql)
@@ -863,7 +863,7 @@ CREATE TABLE IF NOT EXISTS default.test1 (
   `field_1` VARCHAR(255),
   `field_1` VARCHAR(255),
   `field_2` VARCHAR(255),
   `field_2` VARCHAR(255),
   `field_3` bigint);
   `field_3` bigint);
-      
+
 INSERT INTO default.test1 VALUES ('NY', 'New York', '8143197'), ('CA', 'Los Angeles', '3844829'), \
 INSERT INTO default.test1 VALUES ('NY', 'New York', '8143197'), ('CA', 'Los Angeles', '3844829'), \
 ('IL', 'Chicago', '2842518'), ('TX', 'Houston', '2016582'), ('PA', 'Philadelphia', '1463281'), \
 ('IL', 'Chicago', '2842518'), ('TX', 'Houston', '2016582'), ('PA', 'Philadelphia', '1463281'), \
 ('AZ', 'Phoenix', '1461575'), ('TX', 'San Antonio', '1256509'), ('CA', 'San Diego', '1255540'), \
 ('AZ', 'Phoenix', '1461575'), ('TX', 'San Antonio', '1256509'), ('CA', 'San Diego', '1255540'), \
@@ -901,23 +901,23 @@ CREATE TABLE IF NOT EXISTS default.test1 (
 CONSTRAINT my_pk PRIMARY KEY (field_3));
 CONSTRAINT my_pk PRIMARY KEY (field_3));
 
 
 UPSERT INTO default.test1 VALUES ('NY', 'New York', 8143197);
 UPSERT INTO default.test1 VALUES ('NY', 'New York', 8143197);
-            
+
 UPSERT INTO default.test1 VALUES ('CA', 'Los Angeles', 3844829);
 UPSERT INTO default.test1 VALUES ('CA', 'Los Angeles', 3844829);
-            
+
 UPSERT INTO default.test1 VALUES ('IL', 'Chicago', 2842518);
 UPSERT INTO default.test1 VALUES ('IL', 'Chicago', 2842518);
-            
+
 UPSERT INTO default.test1 VALUES ('TX', 'Houston', 2016582);
 UPSERT INTO default.test1 VALUES ('TX', 'Houston', 2016582);
-            
+
 UPSERT INTO default.test1 VALUES ('PA', 'Philadelphia', 1463281);
 UPSERT INTO default.test1 VALUES ('PA', 'Philadelphia', 1463281);
-            
+
 UPSERT INTO default.test1 VALUES ('AZ', 'Phoenix', 1461575);
 UPSERT INTO default.test1 VALUES ('AZ', 'Phoenix', 1461575);
-            
+
 UPSERT INTO default.test1 VALUES ('TX', 'San Antonio', 1256509);
 UPSERT INTO default.test1 VALUES ('TX', 'San Antonio', 1256509);
-            
+
 UPSERT INTO default.test1 VALUES ('CA', 'San Diego', 1255540);
 UPSERT INTO default.test1 VALUES ('CA', 'San Diego', 1255540);
-            
+
 UPSERT INTO default.test1 VALUES ('TX', 'Dallas', 1213825);
 UPSERT INTO default.test1 VALUES ('TX', 'Dallas', 1213825);
-            
+
 UPSERT INTO default.test1 VALUES ('CA', 'San Jose', 912332);'''
 UPSERT INTO default.test1 VALUES ('CA', 'San Jose', 912332);'''
 
 
     assert_equal(statement, sql)
     assert_equal(statement, sql)
@@ -971,7 +971,7 @@ CREATE TABLE IF NOT EXISTS default.test1_tmp (
   `cancelled` string,
   `cancelled` string,
   `time` string,
   `time` string,
   `dist` string);
   `dist` string);
-      
+
 INSERT INTO default.test1_tmp VALUES \
 INSERT INTO default.test1_tmp VALUES \
 ('2011-12-14 12:00:00', '13', '4', '1304', '1704', '24', '14', 'WN', '3085', 'PHL', 'N524SW', '1', '159', '1336'), \
 ('2011-12-14 12:00:00', '13', '4', '1304', '1704', '24', '14', 'WN', '3085', 'PHL', 'N524SW', '1', '159', '1336'), \
 ('2011-12-14 12:00:00', '17', '52', '1752', '1943', '12', '8', 'WN', '39', 'PHX', 'N503SW', '1', '155', '1020'), \
 ('2011-12-14 12:00:00', '17', '52', '1752', '1943', '12', '8', 'WN', '39', 'PHX', 'N503SW', '1', '155', '1020'), \

+ 2 - 2
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -128,6 +128,6 @@ INPUTFORMAT ${table["input_format_class"] | n} OUTPUTFORMAT ${table["output_form
 LOCATION '${table["path"] | n}'
 LOCATION '${table["path"] | n}'
 % endif
 % endif
 % if table.get("tbl_properties"):
 % if table.get("tbl_properties"):
-TBLPROPERTIES(${ ', '.join(['"' + prop + '"' + ' = ' + '"' + table.get("tbl_properties").get(prop) + '"' for prop in table.get("tbl_properties")]) | n })
+TBLPROPERTIES(${ ', '.join(["'" + prop + "'" + '=' + "'" + table.get("tbl_properties").get(prop) + "'" for prop in table.get("tbl_properties")]) | n })
 % endif
 % endif
-;
+;