Browse Source

[importer] adding dropping col feature for small local file

ayush.goyal 4 years ago
parent
commit
8846d432ed

+ 9 - 2
desktop/libs/indexer/src/indexer/indexers/sql.py

@@ -308,7 +308,11 @@ class SQLIndexer(object):
     source_type = source['sourceType']
     source_type = source['sourceType']
     editor_type = destination['sourceType']
     editor_type = destination['sourceType']
 
 
-    columns = destination['columns']
+    cols_to_remove = sorted(
+      [col_index for col_index, col in enumerate(destination['columns']) if not col['keep']],
+      reverse=True
+    )
+    columns = [col for col_index, col in enumerate(destination['columns']) if col_index not in cols_to_remove]
 
 
     dialect = get_interpreter(source_type, self.user)['dialect']
     dialect = get_interpreter(source_type, self.user)['dialect']
 
 
@@ -344,6 +348,8 @@ class SQLIndexer(object):
         for count, row in enumerate(reader):
         for count, row in enumerate(reader):
           if (source['format']['hasHeader'] and count == 0) or not row:
           if (source['format']['hasHeader'] and count == 0) or not row:
             continue
             continue
+          for col_index in cols_to_remove:
+            del row[col_index]
           if dialect == 'impala':                         # for the boolean col updating csv_val to (1,0)
           if dialect == 'impala':                         # for the boolean col updating csv_val to (1,0)
             row = self.nomalize_booleans(row, columns)
             row = self.nomalize_booleans(row, columns)
           _csv_rows.append(tuple(row))
           _csv_rows.append(tuple(row))
@@ -364,7 +370,8 @@ class SQLIndexer(object):
               else '  CAST ( CAST ( `%(name)s` AS TINYINT ) AS boolean ) `%(name)s`' % col for col in columns
               else '  CAST ( CAST ( `%(name)s` AS TINYINT ) AS boolean ) `%(name)s`' % col for col in columns
             ])
             ])
 
 
-            sql += '''\nINSERT INTO %(database)s.%(table_name)s_tmp VALUES %(csv_rows)s;\n\nCREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s
+            sql += '''\nINSERT INTO %(database)s.%(table_name)s_tmp VALUES %(csv_rows)s;\n
+CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s
 AS SELECT\n%(sql_)s\nFROM  %(database)s.%(table_name)s_tmp;\n\nDROP TABLE IF EXISTS %(database)s.%(table_name)s_tmp;'''% {
 AS SELECT\n%(sql_)s\nFROM  %(database)s.%(table_name)s_tmp;\n\nDROP TABLE IF EXISTS %(database)s.%(table_name)s_tmp;'''% {
               'database': database,
               'database': database,
               'table_name': table_name,
               'table_name': table_name,

+ 61 - 31
desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -797,20 +797,20 @@ def test_create_table_from_local():
     destination = {
     destination = {
       'name': 'default.test1',
       'name': 'default.test1',
       'columns': [
       'columns': [
-        {'name': 'date', 'type': 'timestamp'},
-        {'name': 'hour', 'type': 'bigint'},
-        {'name': 'minute', 'type': 'bigint'},
-        {'name': 'dep', 'type': 'bigint'},
-        {'name': 'arr', 'type': 'bigint'},
-        {'name': 'dep_delay', 'type': 'bigint'},
-        {'name': 'arr_delay', 'type': 'bigint'},
-        {'name': 'carrier', 'type': 'string'},
-        {'name': 'flight', 'type': 'bigint'},
-        {'name': 'dest', 'type': 'string'},
-        {'name': 'plane', 'type': 'string'},
-        {'name': 'cancelled', 'type': 'boolean'},
-        {'name': 'time', 'type': 'bigint'},
-        {'name': 'dist', 'type': 'bigint'},
+        {'name': 'date', 'type': 'timestamp', 'keep': True},
+        {'name': 'hour', 'type': 'bigint', 'keep': True},
+        {'name': 'minute', 'type': 'bigint', 'keep': True},
+        {'name': 'dep', 'type': 'bigint', 'keep': True},
+        {'name': 'arr', 'type': 'bigint', 'keep': True},
+        {'name': 'dep_delay', 'type': 'bigint', 'keep': True},
+        {'name': 'arr_delay', 'type': 'bigint', 'keep': True},
+        {'name': 'carrier', 'type': 'string', 'keep': True},
+        {'name': 'flight', 'type': 'bigint', 'keep': True},
+        {'name': 'dest', 'type': 'string', 'keep': True},
+        {'name': 'plane', 'type': 'string', 'keep': True},
+        {'name': 'cancelled', 'type': 'boolean', 'keep': True},
+        {'name': 'time', 'type': 'bigint', 'keep': True},
+        {'name': 'dist', 'type': 'bigint', 'keep': True},
       ],
       ],
       'indexerPrimaryKey': [],
       'indexerPrimaryKey': [],
       'sourceType': 'hive'
       'sourceType': 'hive'
@@ -849,9 +849,9 @@ def test_create_table_from_local_mysql():
     destination = {
     destination = {
       'name': 'default.test1',
       'name': 'default.test1',
       'columns': [
       'columns': [
-        {'name': 'field_1', 'type': 'string'},
-        {'name': 'field_2', 'type': 'string'},
-        {'name': 'field_3', 'type': 'bigint'},
+        {'name': 'field_1', 'type': 'string', 'keep': True},
+        {'name': 'field_2', 'type': 'string', 'keep': True},
+        {'name': 'field_3', 'type': 'bigint', 'keep': True},
       ],
       ],
       'sourceType': 'mysql'
       'sourceType': 'mysql'
     }
     }
@@ -883,20 +883,20 @@ def test_create_table_from_local_impala():
     destination = {
     destination = {
       'name': 'default.test1',
       'name': 'default.test1',
       'columns': [
       'columns': [
-        {'name': 'date', 'type': 'timestamp'},
-        {'name': 'hour', 'type': 'bigint'},
-        {'name': 'minute', 'type': 'bigint'},
-        {'name': 'dep', 'type': 'bigint'},
-        {'name': 'arr', 'type': 'bigint'},
-        {'name': 'dep_delay', 'type': 'bigint'},
-        {'name': 'arr_delay', 'type': 'bigint'},
-        {'name': 'carrier', 'type': 'string'},
-        {'name': 'flight', 'type': 'bigint'},
-        {'name': 'dest', 'type': 'string'},
-        {'name': 'plane', 'type': 'string'},
-        {'name': 'cancelled', 'type': 'boolean'},
-        {'name': 'time', 'type': 'bigint'},
-        {'name': 'dist', 'type': 'bigint'},
+        {'name': 'date', 'type': 'timestamp', 'keep': True},
+        {'name': 'hour', 'type': 'bigint', 'keep': True},
+        {'name': 'minute', 'type': 'bigint', 'keep': True},
+        {'name': 'dep', 'type': 'bigint', 'keep': True},
+        {'name': 'arr', 'type': 'bigint', 'keep': True},
+        {'name': 'dep_delay', 'type': 'bigint', 'keep': True},
+        {'name': 'arr_delay', 'type': 'bigint', 'keep': True},
+        {'name': 'carrier', 'type': 'string', 'keep': True},
+        {'name': 'flight', 'type': 'bigint', 'keep': True},
+        {'name': 'dest', 'type': 'string', 'keep': True},
+        {'name': 'plane', 'type': 'string', 'keep': True},
+        {'name': 'cancelled', 'type': 'boolean', 'keep': True},
+        {'name': 'time', 'type': 'bigint', 'keep': True},
+        {'name': 'dist', 'type': 'bigint', 'keep': True},
       ],
       ],
       'sourceType': 'impala'
       'sourceType': 'impala'
     }
     }
@@ -952,3 +952,33 @@ FROM  default.test1_tmp;
 DROP TABLE IF EXISTS default.test1_tmp;'''
 DROP TABLE IF EXISTS default.test1_tmp;'''
 
 
     assert_equal(statement, sql)
     assert_equal(statement, sql)
+
+
+def test_create_table_with_drop_column_from_local():
+  with patch('indexer.indexers.sql.get_interpreter') as get_interpreter:
+    get_interpreter.return_value = {'Name': 'Hive', 'dialect': 'hive'}
+    source = {
+      'path': '',
+      'sourceType': 'hive'
+    }
+    destination = {
+      'name': 'default.test1',
+      'columns': [
+        {'name': 'date', 'type': 'timestamp', 'keep': False},
+        {'name': 'hour', 'type': 'bigint', 'keep': True},
+        {'name': 'minute', 'type': 'bigint', 'keep': False},
+        {'name': 'dep', 'type': 'bigint', 'keep': True},
+        {'name': 'arr', 'type': 'bigint', 'keep': False},
+      ],
+      'indexerPrimaryKey': [],
+      'sourceType': 'hive'
+    }
+    sql = SQLIndexer(user=Mock(), fs=Mock()).create_table_from_local_file(source, destination).get_str()
+
+    statement = '''USE default;
+
+CREATE TABLE IF NOT EXISTS default.test1 (
+  `hour` bigint,
+  `dep` bigint);'''
+
+    assert_equal(statement, sql)

+ 1 - 1
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -1167,7 +1167,7 @@ ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
 
 
       <span data-bind="visible: showProperties">
       <span data-bind="visible: showProperties">
         <input type="text" class="input-medium margin-left-5" placeholder="${ _('Field comment') }" data-bind="value: comment">
         <input type="text" class="input-medium margin-left-5" placeholder="${ _('Field comment') }" data-bind="value: comment">
-        <label class="checkbox" data-bind="visible: $root.createWizard.destination.tableFormat() == 'kudu'">
+        <label class="checkbox" data-bind="visible: $root.createWizard.destination.tableFormat() == 'kudu' || $root.createWizard.source.inputFormat() == 'localfile'">
           <input type="checkbox" data-bind="checked: keep"> ${_('Keep')}
           <input type="checkbox" data-bind="checked: keep"> ${_('Keep')}
         </label>
         </label>
       </span>
       </span>