Browse Source

HUE-7 [importer] Support creating a kudu table from a text file

Add tests.
Romain Rigaux 8 năm trước cách đây
mục cha
commit
c9591bbed6

+ 7 - 3
desktop/libs/indexer/src/indexer/api3.py

@@ -269,13 +269,17 @@ def _create_table_from_a_file(request, source, destination):
           'database': database,
           'table_name': table_name
       }
-    else:
+    else: # Manual
       row_format = ''
       file_format = table_format
       skip_header = False
       if table_format == 'kudu':
         columns = [col for col in columns if col['name'] in primary_keys] + [col for col in columns if col['name'] not in primary_keys]
 
+  if table_format == 'kudu':
+    collection_delimiter = None
+    map_delimiter = None
+
   if external or (load_data and table_format in ('parquet', 'kudu')):
     if not request.fs.isdir(external_path): # File selected
       external_path, external_file_name = request.fs.split(external_path)
@@ -292,7 +296,7 @@ def _create_table_from_a_file(request, source, destination):
           'row_format': row_format,
           'field_terminator': field_delimiter,
           'collection_terminator': collection_delimiter, # Only if Hive
-          'map_key_terminator': map_delimiter, # Only if Hive # [LINES TERMINATED BY char]
+          'map_key_terminator': map_delimiter, # Only if Hive
           'serde_name': serde_name,
           'serde_properties': serde_properties,
           'file_format': file_format,
@@ -314,7 +318,7 @@ def _create_table_from_a_file(request, source, destination):
   if load_data and table_format in ('parquet', 'kudu'):
     file_format = table_format
     if table_format == 'kudu':
-      columns_list = ['`%s`' % col for col in [primary_keys + [col['name'] for col in destination['columns'] if col['name'] not in primary_keys]]]
+      columns_list = ['`%s`' % col for col in primary_keys + [col['name'] for col in destination['columns'] if col['name'] not in primary_keys]]
       extra_create_properties = """PRIMARY KEY (%(primary_keys)s)
       PARTITION BY HASH PARTITIONS 16
       STORED AS %(file_format)s

Những thai đổi đã bị hủy bỏ vì nó quá lớn
+ 44 - 0
desktop/libs/indexer/src/indexer/create_table_tests.py


+ 3 - 2
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -102,10 +102,11 @@ ROW FORMAT \
 %     if table.has_key('field_terminator'):
     FIELDS TERMINATED BY '${table["field_terminator"] | n}'
 %     endif
-%     if table.has_key('collection_terminator'):
+## [LINES TERMINATED BY char]
+%     if table.get('collection_terminator') is not None:
     COLLECTION ITEMS TERMINATED BY '${table["collection_terminator"] | n}'
 %     endif
-%     if table.has_key('map_key_terminator'):
+%     if table.get('map_key_terminator') is not None:
     MAP KEYS TERMINATED BY '${table["map_key_terminator"] | n}'
 %     endif
 %   else:

Một số tệp đã không được hiển thị bởi vì quá nhiều tập tin thay đổi trong này khác