Jelajahi Sumber

[indexer] Switch to single quotes for TBLPROPERTIES values

The new Hive does not accept those currently.
Romain Rigaux 4 tahun lalu
induk
melakukan
b80a403b67

+ 28 - 33
desktop/libs/indexer/src/indexer/indexers/sql.py

@@ -236,7 +236,7 @@ class SQLIndexer(object):
         PARTITION BY HASH PARTITIONS 16
         STORED AS %(file_format)s
         TBLPROPERTIES(
-        'kudu.num_tablet_replicas' = '1'
+        'kudu.num_tablet_replicas'='1'
         )""" % {
           'file_format': file_format,
           'primary_keys': ', '.join(primary_keys)
@@ -245,7 +245,7 @@ class SQLIndexer(object):
         columns_list = ['*']
         extra_create_properties = 'STORED AS %(file_format)s' % {'file_format': file_format}
         if is_transactional:
-          extra_create_properties += '\nTBLPROPERTIES("transactional"="true", "transactional_properties"="%s")' % \
+          extra_create_properties += "\nTBLPROPERTIES('transactional'='true', 'transactional_properties'='%s')" % \
               default_transactional_type
 
       sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`%(comment)s
@@ -320,12 +320,11 @@ class SQLIndexer(object):
             col['type'] = 'VARCHAR(255)'
 
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
-%(columns)s);
-      ''' % {
-              'database': database,
-              'table_name': table_name,
-              'columns': ',\n'.join(['  `%(name)s` %(type)s' % col for col in columns]),
-            }
+%(columns)s);\n''' % {
+        'database': database,
+        'table_name': table_name,
+        'columns': ',\n'.join(['  `%(name)s` %(type)s' % col for col in columns]),
+      }
 
     elif dialect == 'phoenix':
 
@@ -335,8 +334,7 @@ class SQLIndexer(object):
 
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
 %(columns)s
-CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
-''' % {
+CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));\n''' % {
           'database': database,
           'table_name': table_name,
           'columns': ',\n'.join(['  %(name)s %(type)s' % col for col in columns]),
@@ -345,12 +343,11 @@ CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
 
     elif dialect == 'impala':
       sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s_tmp (
-%(columns)s);
-      ''' % {
-                'database': database,
-                'table_name': table_name,
-                'columns': ',\n'.join(['  `%(name)s` string' % col for col in columns]),
-            }                                                 # Impala does not implicitly cast between string and numeric or Boolean types.
+%(columns)s);\n''' % {
+          'database': database,
+          'table_name': table_name,
+          'columns': ',\n'.join(['  `%(name)s` string' % col for col in columns]),
+      }                                                 # Impala does not implicitly cast between string and numeric or Boolean types.
 
     path = urllib_unquote(source['path'])
 
@@ -370,23 +367,21 @@ CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
           csv_rows = str(_csv_rows)[1:-1]
 
           if dialect in ('hive', 'mysql'):
-            sql += '''\nINSERT INTO %(database)s.%(table_name)s VALUES %(csv_rows)s;
-            '''% {
-                    'database': database,
-                    'table_name': table_name,
-                    'csv_rows': csv_rows
-                  }
+            sql += '''\nINSERT INTO %(database)s.%(table_name)s VALUES %(csv_rows)s;\n'''% {
+              'database': database,
+              'table_name': table_name,
+              'csv_rows': csv_rows
+            }
           elif dialect == 'phoenix':
             for csv_row in _csv_rows:
               _sql = ', '.join([ "'{0}'".format(col_val) if columns[count]['type'] in ('CHAR(255)', 'timestamp') \
                 else '{0}'.format(col_val) for count, col_val in enumerate(csv_row)])
 
-              sql += '''\nUPSERT INTO %(database)s.%(table_name)s VALUES (%(csv_row)s);
-            ''' % {
-                    'database': database,
-                    'table_name': table_name,
-                    'csv_row': _sql
-                  }
+              sql += '''\nUPSERT INTO %(database)s.%(table_name)s VALUES (%(csv_row)s);\n''' % {
+                'database': database,
+                'table_name': table_name,
+                'csv_row': _sql
+              }
           elif dialect == 'impala':
              # casting from string to boolean is not allowed in impala so string -> int -> bool
             sql_ = ',\n'.join([
@@ -396,11 +391,11 @@ CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
 
             sql += '''\nINSERT INTO %(database)s.%(table_name)s_tmp VALUES %(csv_rows)s;\n\nCREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s
 AS SELECT\n%(sql_)s\nFROM  %(database)s.%(table_name)s_tmp;\n\nDROP TABLE IF EXISTS %(database)s.%(table_name)s_tmp;'''% {
-                    'database': database,
-                    'table_name': table_name,
-                    'csv_rows': csv_rows,
-                    'sql_': sql_
-                  }
+              'database': database,
+              'table_name': table_name,
+              'csv_rows': csv_rows,
+              'sql_': sql_
+            }
 
     on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \
         '?source_type=' + source_type

+ 22 - 22
desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -89,14 +89,14 @@ ROW FORMAT   SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
     "quoteChar"     = """,
     "escapeChar"    = "\\\\"
     )
-  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+  STORED AS TextFile TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;
 
 LOAD DATA INPATH 'hdfs:///path/data.csv' INTO TABLE `default`.`hue__tmp_export_table` PARTITION (day='20200101');
 
 CREATE TABLE `default`.`export_table` COMMENT "No comment!"
         STORED AS csv
-TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
+TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only')
         AS SELECT *
         FROM `default`.`hue__tmp_export_table`;
 
@@ -216,7 +216,7 @@ ROW FORMAT   DELIMITED
     FIELDS TERMINATED BY ','
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
-  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+  STORED AS TextFile TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
   assert_true(statement in sql, sql)
 
@@ -334,7 +334,7 @@ def test_generate_create_kudu_table_with_data():
 ROW FORMAT   DELIMITED
     FIELDS TERMINATED BY ','
   STORED AS TextFile LOCATION '/A'
-TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")'''
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')'''
     assert_true(statement in sql, sql)
 
     assert_true('''CREATE TABLE `default`.`index_data` COMMENT "Big Data"
@@ -342,7 +342,7 @@ TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")'''
         PARTITION BY HASH PARTITIONS 16
         STORED AS kudu
         TBLPROPERTIES(
-        'kudu.num_tablet_replicas' = '1'
+        'kudu.num_tablet_replicas'='1'
         )
         AS SELECT `id`, `business_id`, `date`, `funny`, `stars`, `text`, `type`, `useful`, `user_id`, `name`, '''
         '''`full_address`, `latitude`, `longitude`, `neighborhoods`, `open`, `review_count`, `state`
@@ -429,7 +429,7 @@ def test_generate_create_parquet_table():
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
   STORED AS TextFile LOCATION '/user/hue/data'
-TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
   assert_true(statement in sql, sql)
 
@@ -521,13 +521,13 @@ def test_generate_create_orc_table_transactional():
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
   STORED AS TextFile LOCATION '/user/hue/data'
-TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
   assert_true(statement in sql, sql)
 
   assert_true('''CREATE TABLE `default`.`parquet_table`
         STORED AS orc
-TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
+TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only')
         AS SELECT *
         FROM `default`.`hue__tmp_parquet_table`;
 ''' in sql, sql)
@@ -580,7 +580,7 @@ def test_generate_create_empty_kudu_table():
   `vrfcn_city` string ,
   `vrfcn_city_lat` double ,
   `vrfcn_city_lon` double , PRIMARY KEY (acct_client)
-)   STORED AS kudu TBLPROPERTIES("transactional" = "false")
+)   STORED AS kudu TBLPROPERTIES('transactional'='false')
 ;''' in sql, sql)
 
 
@@ -742,7 +742,7 @@ def test_create_ddl_with_nonascii():
     FIELDS TERMINATED BY ','
     COLLECTION ITEMS TERMINATED BY '\\002'
     MAP KEYS TERMINATED BY '\\003'
-  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+  STORED AS TextFile TBLPROPERTIES('skip.header.line.count'='1', 'transactional'='false')
 ;'''
   assert_true(statement in sql, sql)
 
@@ -752,7 +752,7 @@ def test_create_ddl_with_nonascii():
 
   statement = '''CREATE TABLE `default`.`renamed_chinese_cities_gb2312`
         STORED AS TextFile
-TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
+TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only')
         AS SELECT *
         FROM `default`.`hue__tmp_renamed_chinese_cities_gb2312`;'''
   assert_true(statement in sql, sql)
@@ -863,7 +863,7 @@ CREATE TABLE IF NOT EXISTS default.test1 (
   `field_1` VARCHAR(255),
   `field_2` VARCHAR(255),
   `field_3` bigint);
-      
+
 INSERT INTO default.test1 VALUES ('NY', 'New York', '8143197'), ('CA', 'Los Angeles', '3844829'), \
 ('IL', 'Chicago', '2842518'), ('TX', 'Houston', '2016582'), ('PA', 'Philadelphia', '1463281'), \
 ('AZ', 'Phoenix', '1461575'), ('TX', 'San Antonio', '1256509'), ('CA', 'San Diego', '1255540'), \
@@ -901,23 +901,23 @@ CREATE TABLE IF NOT EXISTS default.test1 (
 CONSTRAINT my_pk PRIMARY KEY (field_3));
 
 UPSERT INTO default.test1 VALUES ('NY', 'New York', 8143197);
-            
+
 UPSERT INTO default.test1 VALUES ('CA', 'Los Angeles', 3844829);
-            
+
 UPSERT INTO default.test1 VALUES ('IL', 'Chicago', 2842518);
-            
+
 UPSERT INTO default.test1 VALUES ('TX', 'Houston', 2016582);
-            
+
 UPSERT INTO default.test1 VALUES ('PA', 'Philadelphia', 1463281);
-            
+
 UPSERT INTO default.test1 VALUES ('AZ', 'Phoenix', 1461575);
-            
+
 UPSERT INTO default.test1 VALUES ('TX', 'San Antonio', 1256509);
-            
+
 UPSERT INTO default.test1 VALUES ('CA', 'San Diego', 1255540);
-            
+
 UPSERT INTO default.test1 VALUES ('TX', 'Dallas', 1213825);
-            
+
 UPSERT INTO default.test1 VALUES ('CA', 'San Jose', 912332);'''
 
     assert_equal(statement, sql)
@@ -971,7 +971,7 @@ CREATE TABLE IF NOT EXISTS default.test1_tmp (
   `cancelled` string,
   `time` string,
   `dist` string);
-      
+
 INSERT INTO default.test1_tmp VALUES \
 ('2011-12-14 12:00:00', '13', '4', '1304', '1704', '24', '14', 'WN', '3085', 'PHL', 'N524SW', '1', '159', '1336'), \
 ('2011-12-14 12:00:00', '17', '52', '1752', '1943', '12', '8', 'WN', '39', 'PHX', 'N503SW', '1', '155', '1020'), \

+ 2 - 2
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -128,6 +128,6 @@ INPUTFORMAT ${table["input_format_class"] | n} OUTPUTFORMAT ${table["output_form
 LOCATION '${table["path"] | n}'
 % endif
 % if table.get("tbl_properties"):
-TBLPROPERTIES(${ ', '.join(['"' + prop + '"' + ' = ' + '"' + table.get("tbl_properties").get(prop) + '"' for prop in table.get("tbl_properties")]) | n })
+TBLPROPERTIES(${ ', '.join(["'" + prop + "'" + '=' + "'" + table.get("tbl_properties").get(prop) + "'" for prop in table.get("tbl_properties")]) | n })
 % endif
-;
+;