Ver código fonte

HUE-6965 [importer] Supporting import in different file types with different delimeters

Prachi Poddar 8 anos atrás
pai
commit
7fa5513

+ 19 - 0
desktop/libs/indexer/src/indexer/indexers/rdbms.py

@@ -115,6 +115,10 @@ def run_sqoop(request, source, destination, start_time):
   destination_table_name = destination['tableName']
   destination_table_name = destination['tableName']
   destination_database_name = destination['databaseName']
   destination_database_name = destination['databaseName']
   destination_mappers_num = destination['numMappers']
   destination_mappers_num = destination['numMappers']
+  destination_file_output_format = destination['rdbmsFileOutputFormat']
+  destination_custom_fields_delimiter = destination['customFieldsDelimiter']
+  destination_custom_line_delimiter = destination['customLineDelimiter']
+  destination_custom_enclosed_by_delimiter = destination['customEnclosedByDelimiter']
 
 
   if not rdbms_all_tables_selected:
   if not rdbms_all_tables_selected:
     rdbms_table_name = source['rdbmsTableName']
     rdbms_table_name = source['rdbmsTableName']
@@ -169,6 +173,21 @@ def run_sqoop(request, source, destination, start_time):
         'rdbmsTableName': rdbms_table_name,
         'rdbmsTableName': rdbms_table_name,
         'targetDir': targetDir
         'targetDir': targetDir
       }
       }
+      if destination_file_output_format == 'text':
+        statement = '%(statement)s --as-textfile --fields-terminated-by %(customFieldsDelimiter)s --lines-terminated-by %(customLineDelimiter)s --enclosed-by %(customEnclosedByDelimiter)s' % {
+          'statement': statement,
+          'customFieldsDelimiter': destination_custom_fields_delimiter,
+          'customLineDelimiter': destination_custom_line_delimiter,
+          'customEnclosedByDelimiter': destination_custom_enclosed_by_delimiter
+        }
+      elif destination_file_output_format == 'sequence':
+        statement = '%(statement)s --as-sequencefile' % {
+          'statement': statement
+        }
+      elif destination_file_output_format == 'avro':
+        statement = '%(statement)s --as-avrodatafile' % {
+          'statement': statement
+        }
   elif destination_type == 'table':
   elif destination_type == 'table':
     success_url = reverse('metastore:describe_table', kwargs={'database': destination_database_name, 'table': destination_table_name})
     success_url = reverse('metastore:describe_table', kwargs={'database': destination_database_name, 'table': destination_table_name})
     if rdbms_all_tables_selected:
     if rdbms_all_tables_selected:

+ 47 - 8
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -657,6 +657,7 @@ ${ assist.assistPanel() }
               <input type="number" class="form-control input-small" data-bind="value: numMappers, valueUpdate: 'afterkeydown'">
               <input type="number" class="form-control input-small" data-bind="value: numMappers, valueUpdate: 'afterkeydown'">
             </label>
             </label>
           </div>
           </div>
+
           <div class="card-body">
           <div class="card-body">
             <label class="control-label"><div>${ _('Libs') }</div>
             <label class="control-label"><div>${ _('Libs') }</div>
               <div class="inline-table">
               <div class="inline-table">
@@ -677,6 +678,34 @@ ${ assist.assistPanel() }
               </div>
               </div>
             </label>
             </label>
           </div>
           </div>
+
+          <div class="control-group">
+            <label class="control-label"><div>${ _('Extras') }</div>
+              <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, click: function() { showProperties(!showProperties()) }" title="${ _('Show extra properties') }">
+                <i class="fa fa-sliders fa-padding-top"></i>
+              </a>
+            </label>
+          </div>
+
+          <span data-bind="visible: showProperties">
+            <div class="control-group" data-bind="visible: outputFormat() == 'file' && !$root.createWizard.source.rdbmsAllTablesSelected()">
+              <label for="destinationFormat" class="control-label"><div>${ _('Format') }</div>
+                <select id="destinationFormat" data-bind="selectize: rdbmsFileOutputFormats, value: rdbmsFileOutputFormat, optionsValue: 'value', optionsText: 'name'"></select>
+              </label>
+            </div>
+            <span class="inline-labels" data-bind="visible: rdbmsFileOutputFormat() == 'text' && outputFormat() == 'file' && !$root.createWizard.source.rdbmsAllTablesSelected()">
+              <label for="fieldDelimiter" class="control-label"><div>${ _('Fields') }</div>
+                <select id="fieldDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customFieldsDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
+              </label>
+              <label for="collectionDelimiter" class="control-label"><div>${ _('Line') }</div>
+                <select id="collectionDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customLineDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
+              </label>
+              <label for="structDelimiter" class="control-label"><div>${ _('Optionally Enclosed By') }</div>
+                <select id="structDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customEnclosedByDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
+              </label>
+            </span>
+          </span>
+
         </div>
         </div>
         <!-- /ko -->
         <!-- /ko -->
 
 
@@ -1696,6 +1725,15 @@ ${ assist.assistPanel() }
         self.sqoopJobLibPaths.remove(valueToRemove);
         self.sqoopJobLibPaths.remove(valueToRemove);
       };
       };
       self.numMappers = ko.observable(1);
       self.numMappers = ko.observable(1);
+      self.customFieldsDelimiter = ko.observable(',');
+      self.customLineDelimiter = ko.observable('\\n');
+      self.customEnclosedByDelimiter = ko.observable('\'');
+      self.rdbmsFileOutputFormat = ko.observable('text');
+      self.rdbmsFileOutputFormats = ko.observableArray([
+          {'value': 'text', 'name': 'text'},
+          {'value': 'sequence', 'name': 'sequence'},
+          {'value': 'avro', 'name': 'avro'}
+      ]);
 
 
     };
     };
 
 
@@ -1739,14 +1777,15 @@ ${ assist.assistPanel() }
       self.destination = new Destination(vm, self);
       self.destination = new Destination(vm, self);
 
 
       self.customDelimiters = ko.observableArray([
       self.customDelimiters = ko.observableArray([
-        {'value': ',', 'name': 'Comma (,)'},
-        {'value': '\\t', 'name': '^Tab (\\t)'},
-        {'value': '\\n', 'name': 'New line'},
-        {'value': ' ', 'name': 'Space'},
-        {'value': '"', 'name': 'Double Quote'},
-        {'value': '\\001', 'name': '^A (\\001)'},
-        {'value': '\\002', 'name': '^B (\\002)'},
-        {'value': '\\003', 'name': '^C (\\003)'},
+        {'value': ',', 'name': '${ _("Comma (,)") }'},
+        {'value': '\\t', 'name': '${ _("^Tab (\\t)") }'},
+        {'value': '\\n', 'name': '${ _("New line") }'},
+        {'value': '|', 'name': '${ _("Pipe") }'},
+        {'value': '\"', 'name': '${ _("Double Quote") }'},
+        {'value': '\'', 'name': '${ _("Single Quote") }'},
+        {'value': '\\001', 'name': '${ _("^A (\\001)") }'},
+        {'value': '\\002', 'name': '${ _("^B (\\002)") }'},
+        {'value': '\\003', 'name': '${ _("^C (\\003)") }'},
       ]);
       ]);
 
 
       self.editorId = ko.observable();
       self.editorId = ko.observable();