Răsfoiți Sursa

HUE-5757 [importer] Add CSV Table format

Only works with Hive and convert all column types to 'string'.
In Follow-up, can be use as cleaning-step to Parquet or Kudu table.
Romain Rigaux 8 ani în urmă
părinte
comite
9d243e7f22

+ 11 - 6
desktop/libs/indexer/src/indexer/api3.py

@@ -250,15 +250,20 @@ def _create_table_from_a_file(request, source, destination):
 
   if table_format == 'json':
     row_format = 'serde'
-    serde_name = 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
-    serde_properties = '''"separatorChar" = "\\t",
-   "quoteChar"     = "'",
-   "escapeChar"    = "\\\\"
-   '''
+    serde_name = 'org.apache.hive.hcatalog.data.JsonSerDe'
   elif table_format == 'regexp':
     row_format = 'serde'
     serde_name = 'org.apache.hadoop.hive.serde2.RegexSerDe'
     serde_properties = '"input.regex" = "%s"' % regexp_delimiter
+  elif table_format == 'csv':
+    if source['format']['quoteChar'] == '"':
+      source['format']['quoteChar'] = '\\"'
+    row_format = 'serde'
+    serde_name = 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+    serde_properties = '''"separatorChar" = "%(fieldSeparator)s",
+  "quoteChar"     = "%(quoteChar)s",
+  "escapeChar"    = "\\\\"
+  ''' % source['format']
 
 
   if table_format in ('parquet', 'kudu'):
@@ -312,7 +317,7 @@ def _create_table_from_a_file(request, source, destination):
     }
   )
 
-  if table_format == 'text' and not external and load_data:
+  if table_format in ('text', 'json', 'csv', 'regexp') and not external and load_data:
     sql += "\n\nLOAD DATA INPATH '%s' INTO TABLE `%s`.`%s`;" % (source_path, database, table_name)
 
   if load_data and table_format in ('parquet', 'kudu'):

+ 10 - 12
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -29,7 +29,7 @@
 %if not is_embeddable:
 ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
 
-# Todo lot of those
+## TODO lot of those re-imported
 <script src="${ static('desktop/js/autocomplete/sql.js') }"></script>
 <script src="${ static('desktop/js/sqlAutocompleter.js') }"></script>
 <script src="${ static('desktop/js/sqlAutocompleter2.js') }"></script>
@@ -538,12 +538,6 @@ ${ assist.assistPanel() }
               </label>
             </div>
 
-            <div class="control-group">
-              <label class="checkbox" data-bind="visible: $root.createWizard.source.inputFormat() != 'manual'">
-                <input type="checkbox" data-bind="checked: importData, disable: ! useDefaultLocation() && $parent.createWizard.source.path() == nonDefaultLocation();"> ${_('Import data')}
-              </label>
-            </div>
-
             <div class="control-group">
               <label class="control-label"><div>${ _('Extras') }</div>
                 <a href="javascript:void(0)" data-bind="css: {'inactive-action': !showProperties()}, click: function() {showProperties(!showProperties()) }" title="${ _('Show extra properties') }">
@@ -551,7 +545,13 @@ ${ assist.assistPanel() }
                 </a>
               </label>
             </div>
+
             <span data-bind="visible: showProperties">
+              <div class="control-group">
+                <label class="checkbox" data-bind="visible: $root.createWizard.source.inputFormat() != 'manual'">
+                  <input type="checkbox" data-bind="checked: importData, disable: ! useDefaultLocation() && $parent.createWizard.source.path() == nonDefaultLocation();"> ${_('Import data')}
+                </label>
+              </div>
               <div class="control-group">
                 <label><div>${ _('Description') }</div>
                    <input type="text" class="form-control input-xxlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
@@ -1181,7 +1181,7 @@ ${ assist.assistPanel() }
         }
 
         if (typeof newVal.fieldSeparator !== 'undefined') {
-          vm.createWizard.destination.useCustomDelimiters(true);
+          vm.createWizard.destination.useCustomDelimiters(newVal.fieldSeparator() != ',');
           vm.createWizard.destination.customFieldDelimiter(newVal.fieldSeparator());
           newVal.fieldSeparator.subscribe(function(newVal) {
             vm.createWizard.destination.customFieldDelimiter(newVal);
@@ -1356,13 +1356,11 @@ ${ assist.assistPanel() }
       self.tableFormats = ko.observableArray([
           {'value': 'text', 'name': 'Text'},
           {'value': 'parquet', 'name': 'Parquet'},
-          {'value': 'json', 'name': 'Json'},
           {'value': 'kudu', 'name': 'Kudu'},
-          {'value': 'avro', 'name': 'Avro'},
+          {'value': 'csv', 'name': 'Csv'},
+          {'value': 'json', 'name': 'Json'},
           {'value': 'regexp', 'name': 'Regexp'},
-          {'value': 'rcfile', 'name': 'RCFile'},
           {'value': 'orc', 'name': 'ORC'},
-          {'value': 'sequencefile', 'name': 'SequenceFile'}
       ]);
 
       self.partitionColumns = ko.observableArray();