Bladeren bron

HUE-7 [metastore] Add Kudu as a supported table output format

Romain Rigaux 9 jaren geleden
bovenliggende
commit
f4c133f
2 gewijzigde bestanden met toevoegingen van 25 en 16 verwijderingen
  1. 12 13
      desktop/libs/indexer/src/indexer/api3.py
  2. 13 3
      desktop/libs/indexer/src/indexer/templates/importer.mako

+ 12 - 13
desktop/libs/indexer/src/indexer/api3.py

@@ -178,6 +178,7 @@ def _create_table_from_a_file(request, source, destination):
 
   comment = destination['description']
 
+  source_path = source['path']
   external = not destination['useDefaultLocation']
   external_path = not destination['nonDefaultLocation']
 
@@ -197,26 +198,24 @@ def _create_table_from_a_file(request, source, destination):
     map_delimiter = r'\\003'
     regexp_delimiter = '.*'
 
-  source_path = source['path']
-
 
   file_format = 'TextFile'
   extra_create_properties = ''
   sql = ''
 
-  # if external and non text and load_data, both bath !=
-
   if load_data:
     if table_format in ('parquet', 'kudu'):
-      table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name # Or tmp table?
+      table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name
 
-  if external:
-    if not request.fs.isdir(source_path): # File selected
-      source_path = request.fs.split(source_path)[0]
-    # If dir not empty, create data dir %(filename)_table and move file there...
+  if external or load_data and table_format in ('parquet', 'kudu'):
+    if not request.fs.isdir(external_path): # File selected
+      external_path = request.fs.split(external_path)[0]
+      if len(request.fs.listdir(external_path)) != 1:
+        external_path = external_path + '_table' # If dir not just the file, create data dir and move file there.
+        request.fs.mkdir(source_path + '_table')
+        request.fs.rename(source_path, external_path)
 
-    # Guess format should accept a directory too
-  # Kudu external table has extra prop
+  
   sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
       'database': database,
       'table_name': table_name
@@ -229,8 +228,8 @@ def _create_table_from_a_file(request, source, destination):
           'row_format': 'Delimited',
           'field_terminator': field_delimiter,
           'file_format': file_format,
-          'external': external,
-          'path': source_path,
+          'external': external or load_data and table_format in ('parquet', 'kudu'),
+          'path': external_path,
           'skip_header': skip_header,
           'primary_keys': primary_keys if table_format == 'kudu' and not load_data else []
        },

+ 13 - 3
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -354,7 +354,6 @@ ${ assist.assistPanel() }
             <input type="text" data-bind="value: database">
           </label>
 
-
           <label class="checkbox">
             <input type="checkbox" data-bind="checked: importData, disable: ! useDefaultLocation() && $parent.createWizard.source.path() == nonDefaultLocation();"> ${_('Import data')}
           </label>
@@ -390,6 +389,13 @@ ${ assist.assistPanel() }
             <input type="text" data-bind="value: customRegexp"> ${_('Regexp')}
           </span>
 
+          <div class="control-group" data-bind="visible: tableFormat() == 'kudu'">
+            <label for="kuduPks" class="control-label"><div>${ _('Primary keys') }</div>
+              ## At least one selected
+              <select id="kuduPks" data-bind="options: columns, selectedOptions: primaryKeys, optionsValue: 'name', optionsText: 'name'" size="3" multiple="true"></select>
+            </label>
+          </div>
+
           <div class="row" style="margin-left: 8px">
             <div class="span3">
               <input type="checkbox" data-bind="checked: hasHeader">
@@ -480,7 +486,7 @@ ${ assist.assistPanel() }
 
   <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Nested')}</a>
   <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
-  <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Comment')}</a>
+  ${_('Comment')}
 </script>
 
 
@@ -891,7 +897,6 @@ ${ assist.assistPanel() }
           {'value': 'text', 'name': 'Text'},
           {'value': 'parquet', 'name': 'Parquet'},
           {'value': 'json', 'name': 'Json'},
-          {'value': 'orc', 'name': 'ORC'},
           {'value': 'kudu', 'name': 'Kudu'}
       ]);
       self.ouputFormat = ko.observable('table');
@@ -1090,6 +1095,11 @@ ${ assist.assistPanel() }
                 self.isIndexing(false);
                 self.indexingStarted(false);
                 self.indexingError(true);
+              } else {
+                // if success of CREATE and one more DROP statement execute
+                if (self.editorVM.selectedNotebook().snippets()[0].result.handle().statement_id < self.editorVM.selectedNotebook().snippets()[0].result.handle().statements_count) {
+                  self.editorVM.selectedNotebook().snippets()[0].execute();
+                }
               }
             });
             self.editorVM.selectedNotebook().snippets()[0].checkStatus();