Эх сурвалжийг харах

HUE-6993 [sqoop] Support for configurable number of mappers

Prachi Poddar 8 жил өмнө
parent
commit
0dfe47f

+ 7 - 2
desktop/libs/indexer/src/indexer/indexers/rdbms.py

@@ -78,6 +78,7 @@ def run_sqoop(request, source, destination, start_time):
   destination_name = destination['name']
   destination_table_name = destination['tableName']
   destination_database_name = destination['databaseName']
+  destination_mappers_num = destination['numMappers']
 
   if not rdbms_all_tables_selected:
     rdbms_table_name = source['rdbmsTableName']
@@ -105,12 +106,12 @@ def run_sqoop(request, source, destination, start_time):
     success_url = '/filebrowser/view/' + destination_name
     targetDir = request.fs.fs_defaultfs + destination_name
     if rdbms_all_tables_selected:
-      statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s -m 1' % {
+      statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s' % {
         'statement': statement,
         'targetDir': targetDir
       }
     else:
-      statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s -m 1' % {
+      statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s' % {
         'statement': statement,
         'rdbmsTableName': rdbms_table_name,
         'targetDir': targetDir
@@ -129,6 +130,10 @@ def run_sqoop(request, source, destination, start_time):
   elif destination_type == 'hbase':
     success_url = '/hbase/#HBase/' + destination_table_name
     # Todo
+  statement = '%(statement)s --num-mappers %(numMappers)s' % {
+    'statement': statement,
+    'numMappers': destination_mappers_num
+  }
 
   lib_files = []
   if destination['sqoopJobLibPaths']:

+ 8 - 1
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -418,7 +418,7 @@ ${ assist.assistPanel() }
       </div>
 
 
-        <!-- ko if: outputFormat() == 'table' -->
+        <!-- ko if: outputFormat() == 'table' && $root.createWizard.source.inputFormat() != 'rdbms' -->
         <div class="card step">
           <h4>${_('Properties')}</h4>
           <div class="card-body">
@@ -633,6 +633,11 @@ ${ assist.assistPanel() }
         <!-- ko if: $root.createWizard.source.inputFormat() == 'rdbms' && ['file', 'table', 'hbase'].indexOf(outputFormat()) != -1 -->
         <div class="card step">
           <h4>${_('Properties')}</h4>
+          <div class="control-group">
+            <label><div>${ _('Mappers') }</div>
+              <input type="number" class="form-control input-small" data-bind="value: numMappers, valueUpdate: 'afterkeydown'">
+            </label>
+          </div>
           <div class="card-body">
             <label class="control-label"><div>${ _('Libs') }</div>
               <div class="inline-table">
@@ -1633,6 +1638,8 @@ ${ assist.assistPanel() }
       self.removeSqoopJobLibPath = function (valueToRemove) {
         self.sqoopJobLibPaths.remove(valueToRemove);
       };
+      self.numMappers = ko.observable(1);
+
     };
 
     var CreateWizard = function (vm) {