Bladeren bron

HUE-8580 [importer] Improve usability of table import

jdesjean 7 jaren geleden
bovenliggende
commit
b49e98c

+ 14 - 9
desktop/libs/indexer/src/indexer/indexers/rdbms.py

@@ -66,25 +66,25 @@ def _get_api(request):
   if file_format['rdbmsMode'] == 'customRdbms':
     type = 'custom'
     if file_format['rdbmsType'] == 'jdbc':
-      name = file_format['rdbmsHostname']
+      name = file_format['rdbmsHostname'] # We make sure it's unique as name is the cache key
       interface = file_format['rdbmsType']
       options = {'driver': file_format['rdbmsJdbcDriver'],
                  'url': file_format['rdbmsHostname'],
                  'user': file_format['rdbmsUsername'],
-                  'password': file_format['rdbmsPassword']
+                 'password': file_format['rdbmsPassword']
                 }
     else:
       interface = 'rdbms'
       query_server = {
         'server_name': file_format['rdbmsType'],
         'server_host': file_format['rdbmsHostname'],
-        'server_port': int(file_format['rdbmsPort'] or 3306),
+        'server_port': int(file_format['rdbmsPort'] or '3306'),
         'username': file_format['rdbmsUsername'],
         'password': file_format['rdbmsPassword'],
         'options': {},
         'alias': file_format['rdbmsType']
       }
-      name = query_server['server_host'] + ':' + query_server['port']
+      name = 'rdbms:%(server_name)s://%(server_host)s:%(server_port)s' % query_server # We make sure it's unique as name is the cache key
   else:
     type = file_format['rdbmsJdbcDriverName'] and file_format['rdbmsJdbcDriverName'].lower()
     name = type
@@ -155,11 +155,18 @@ def run_sqoop(request, source, destination, start_time):
       rdbms_port = DATABASES[rdbms_name].PORT.get()
       rdbms_user_name = DATABASES[rdbms_name].USER.get()
       rdbms_password = get_database_password(rdbms_name)
+      url = "jdbc:%(rdbmsType)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s" % {
+        'rdbmsType': rdbms_name,
+        'rdbmsHost': rdbms_host,
+        'rdbmsPort': rdbms_port,
+        'rdbmsDatabaseName': rdbms_database_name,
+      }
     else:
       rdbms_host = source['rdbmsHostname']
       rdbms_port = source['rdbmsPort']
       rdbms_user_name = source['rdbmsUsername']
       rdbms_password = source['rdbmsPassword']
+      url = rdbms_host
 
     password_file_path = request.fs.join(request.fs.get_home_dir() + '/sqoop/', uuid.uuid4().hex + '.password')
     request.fs.do_as_user(request.user, request.fs.create, password_file_path, overwrite=True, permission=0700, data=smart_str(rdbms_password))
@@ -168,10 +175,8 @@ def run_sqoop(request, source, destination, start_time):
     if destination['sqoopJobLibPaths']:
       lib_files = [{'path': f['path'], 'type': 'jar'} for f in destination['sqoopJobLibPaths'] if f['path']]
 
-    statement = '--connect jdbc:%(rdbmsType)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password-file %(passwordFilePath)s' % {
-      'rdbmsType': rdbms_name,
-      'rdbmsHost': rdbms_host,
-      'rdbmsPort': rdbms_port,
+    statement = '--connect %(url)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password-file %(passwordFilePath)s' % {
+      'url': url,
       'rdbmsDatabaseName': rdbms_database_name,
       'rdbmsUserName': rdbms_user_name,
       'passwordFilePath': password_file_path
@@ -231,7 +236,7 @@ def run_sqoop(request, source, destination, start_time):
     success_url = '/hbase/#HBase/' + destination_table_name
 
     # Todo
-  statement = '%(statement)s --num-mappers %(numMappers)s' % {
+  statement = '%(statement)s --num-mappers %(numMappers)s --verbose' % {
     'statement': statement,
     'numMappers': destination_mappers_num
   }

+ 23 - 0
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -318,6 +318,7 @@ ${ assist.assistPanel() }
               <div class="control-group">
                 <label for="rdbmsJdbcDriverName" class="control-label"><div>${ _('Options') }</div>
                   <select id="rdbmsJdbcDriverName" data-bind="selectize: createWizard.source.rdbmsJdbcDriverNames, value: createWizard.source.rdbmsJdbcDriverName, optionsText: 'name', optionsValue: 'value'"></select>
+                  <!-- ko hueSpinner: { spin: createWizard.source.isFetchingDriverNames, inline: true } --><!-- /ko -->
                 </label>
               </div>
               <!-- /ko -->
@@ -326,6 +327,7 @@ ${ assist.assistPanel() }
               <div class="control-group input-append">
                 <label for="rdbmsDatabaseName" class="control-label"><div>${ _('Database Name') }</div>
                   <select id="rdbmsDatabaseName" data-bind="selectize: createWizard.source.rdbmsDatabaseNames, value: createWizard.source.rdbmsDatabaseName, optionsText: 'name', optionsValue: 'value'"></select>
+                  <!-- ko hueSpinner: { spin: createWizard.source.isFetchingDatabaseNames, inline: true } --><!-- /ko -->
                 </label>
               </div>
               <!-- /ko -->
@@ -340,6 +342,7 @@ ${ assist.assistPanel() }
                 <!-- ko ifnot: createWizard.source.rdbmsIsAllTables() -->
                 <label for="rdbmsTableName" class="control-label"><div>${ _('Table Name') }</div>
                   <select id="rdbmsTableName" multiple="multiple" data-bind="selectize: createWizard.source.rdbmsTableNames, selectedObjects: createWizard.source.tables, selectedOptions: createWizard.source.tablesNames, optionsText: 'name', optionsValue: 'value'"></select>
+                  <!-- ko hueSpinner: { spin: createWizard.source.isFetchingTableNames, inline: true } --><!-- /ko -->
                 </label>
                 <!-- /ko -->
               </div>
@@ -1662,26 +1665,33 @@ ${ assist.assistPanel() }
         self.path('');
         resizeElements();
         if(self.rdbmsMode() === 'configRdbms' && val !== 'jdbc') {
+          self.isFetchingDatabaseNames(true);
           $.post("${ url('indexer:get_db_component') }", {
             "source": ko.mapping.toJSON(self)
           }, function (resp) {
             if (resp.data) {
               self.rdbmsDatabaseNames(resp.data);
             }
+          }).always(function(){
+            self.isFetchingDatabaseNames(false);
           });
         } else if(self.rdbmsMode() === 'configRdbms' && val === 'jdbc') {
+          self.isFetchingDriverNames(true);
           $.post("${ url('indexer:jdbc_db_list') }", {
             "source": ko.mapping.toJSON(self)
           }, function (resp) {
             if (resp.data) {
               self.rdbmsJdbcDriverNames(resp.data);
             }
+          }).always(function() {
+            self.isFetchingDriverNames(false);
           });
         }
       });
       self.rdbmsDatabaseName = ko.observable('');
       self.rdbmsDatabaseName.subscribe(function (val) {
         if (val !== '') {
+          self.isFetchingTableNames(true);
           $.post("${ url('indexer:get_db_component') }", {
             "source": ko.mapping.toJSON(self)
           }, function (resp) {
@@ -1690,26 +1700,34 @@ ${ assist.assistPanel() }
                 return ko.mapping.fromJS(opt);
               }));
             }
+          }).always(function(){
+            self.isFetchingTableNames(false);
           });
         }
       });
       self.rdbmsDatabaseNames = ko.observableArray();
+      self.isFetchingDatabaseNames = ko.observable(false);
       self.rdbmsJdbcDriverNames = ko.observableArray();
+      self.isFetchingDriverNames = ko.observable(false);
       self.rdbmsJdbcDriverName = ko.observable();
       self.rdbmsJdbcDriverName.subscribe(function () {
         self.rdbmsDatabaseNames([]);
+        self.isFetchingDatabaseNames(true);
         $.post("${ url('indexer:get_db_component') }", {
           "source": ko.mapping.toJSON(self)
         }, function (resp) {
           if (resp.data) {
             self.rdbmsDatabaseNames(resp.data);
           }
+        }).always(function(){
+          self.isFetchingDatabaseNames(false);
         });
       });
       self.rdbmsJdbcDriver = ko.observable('');
       self.rdbmsJdbcDriver.subscribe(function () {
         self.rdbmsDatabaseNames([]);
       });
+      self.isFetchingTableNames = ko.observable(false);
       self.rdbmsTableNames = ko.observableArray();
       self.rdbmsHostname = ko.observable('');
       self.rdbmsHostname.subscribe(function () {
@@ -1756,6 +1774,8 @@ ${ assist.assistPanel() }
       });
       self.rdbmsDbIsValid = ko.observable(false);
       self.rdbmsCheckConnection = function() {
+        self.isFetchingDatabaseNames(true);
+        self.rdbmsDatabaseName(''); // Need to clear or else get_db_component will return list of tables
         $.post("${ url('indexer:get_db_component') }", {
           "source": ko.mapping.toJSON(self)
         }, function (resp) {
@@ -1766,6 +1786,8 @@ ${ assist.assistPanel() }
             $(document).trigger("error", "${ _('Connection Failed: ') }" + resp.message);
             self.rdbmsDbIsValid(false);
           }
+        }).always(function(){
+          self.isFetchingDatabaseNames(false);
         });
       };
 
@@ -2511,6 +2533,7 @@ ${ assist.assistPanel() }
           self.isGuessingFieldTypes(false);
           guessFieldTypesXhr = null;
         }).fail(function (xhr) {
+          self.loadSampleData({sample_cols: [], columns: [], sample: []});
           $(document).trigger("error", xhr.responseText);
           self.isGuessingFieldTypes(false);
           viewModel.isLoading(false);

+ 2 - 2
desktop/libs/notebook/src/notebook/connectors/jdbc.py

@@ -166,7 +166,7 @@ class JdbcApi(Api):
 
     sample_data, description = assist.get_sample_data(database, table, column)
 
-    if sample_data:
+    if sample_data or description:
       response['status'] = 0
       response['headers'] = [col[0] for col in description] if description else []
       response['full_headers'] = [{
@@ -174,7 +174,7 @@ class JdbcApi(Api):
         'type': col[1],
         'comment': ''
       } for col in description]
-      response['rows'] = sample_data
+      response['rows'] = sample_data if sample_data else []
     else:
       response['message'] = _('Failed to get sample data.')