Эх сурвалжийг харах

HUE-8538 [sqoop] Allow table preview from manual input not JDBC

Romain Rigaux 7 жил өмнө
parent
commit
0273da6

+ 8 - 4
desktop/libs/indexer/src/indexer/api3.py

@@ -42,7 +42,7 @@ from indexer.file_format import HiveFormat
 from indexer.fields import Field
 from indexer.indexers.envelope import EnvelopeIndexer
 from indexer.indexers.morphline import MorphlineIndexer
-from indexer.indexers.rdbms import RdbmsIndexer, run_sqoop
+from indexer.indexers.rdbms import RdbmsIndexer, run_sqoop,  _get_db
 from indexer.indexers.sql import SQLIndexer
 from indexer.solr_client import SolrClient, MAX_UPLOAD_SIZE
 
@@ -210,9 +210,13 @@ def guess_field_types(request):
         "columns": columns,
     }
   elif file_format['inputFormat'] == 'rdbms':
-    query_server = rdbms.get_query_server_config(server=file_format['rdbmsType'])
-    db = rdbms.get(request.user, query_server=query_server)
-    sample = RdbmsIndexer(request.user, file_format['rdbmsType']).get_sample_data(mode=file_format['rdbmsMode'], database=file_format['rdbmsDatabaseName'], table=file_format['rdbmsTableName'])
+    if file_format.get('rdbmsUsername'):
+      db = _get_db(request)
+    else:
+      query_server = rdbms.get_query_server_config(server=file_format['rdbmsType'])
+      db = rdbms.get(request.user, query_server=query_server)
+
+    sample = RdbmsIndexer(request.user, file_format['rdbmsType'], db=db).get_sample_data(mode=file_format['rdbmsMode'], database=file_format['rdbmsDatabaseName'], table=file_format['rdbmsTableName'])
     table_metadata = db.get_columns(file_format['rdbmsDatabaseName'], file_format['rdbmsTableName'], names_only=False)
 
     # cf. https://github.com/apache/sqoop/blob/trunk/src/java/org/apache/sqoop/hive/HiveTypes.java#L39

+ 1 - 0
desktop/libs/indexer/src/indexer/conf.py

@@ -136,6 +136,7 @@ CONFIG_INDEXING_TEMPLATES_PATH = Config(
   default=os.path.join(os.path.dirname(__file__), '..', 'data', 'oozie_workspace')
   )
 
+# Unused
 SOLRCTL_PATH = Config(
   key="solrctl_path",
   help=_t("Location of the solrctl binary."),

+ 49 - 35
desktop/libs/indexer/src/indexer/indexers/rdbms.py

@@ -43,42 +43,15 @@ def get_db_component(request):
 
   try:
     source = json.loads(request.POST.get('source', '{}'))
-    user = User.objects.get(username=request.user)
-    name = None
-    if source['rdbmsMode'] == 'configRdbms':
-      if source['rdbmsType'] != 'jdbc':
-        query_server = rdbms.get_query_server_config(server=source['rdbmsType'])
-        db = rdbms.get(user, query_server=query_server)
-      else:
-        interpreters = get_ordered_interpreters(request.user)
-        options = {}
-        key = [key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName']]
-        if key:
-          options = key[0]['options']
-
-          db = Jdbc(driver_name=options['driver'], url=options['url'], username=options['user'], password=options['password'])
-    else:
-      name = source['rdbmsType']
-      if name != 'jdbc':
-        query_server = {
-          'server_name': name,
-          'server_host': source['rdbmsHostname'],
-          'server_port': int(source['rdbmsPort']),
-          'username': source['rdbmsUsername'],
-          'password': source['rdbmsPassword'],
-          'options': {},
-          'alias': name
-        }
-        db = rdbms.get(user, query_server=query_server)
-      else:
-        db = Jdbc(driver_name=source['rdbmsJdbcDriver'], url=source['rdbmsHostname'], username=source['rdbmsUsername'], password=source['rdbmsPassword'])
+
+    db = _get_db(request)
 
     if source['rdbmsType'] != 'jdbc':
       assist = Assist(db)
     else:
       assist = JdbcAssist(db)
 
-    if not source['rdbmsDatabaseName']:
+    if not source['rdbmsDatabaseName'] or (source['rdbmsMode'] == "customRdbms" and not source['rdbmsDbIsValid']):
       data = assist.get_databases()
     elif source['rdbmsDatabaseName']:
       data = assist.get_tables(source['rdbmsDatabaseName'])
@@ -86,12 +59,47 @@ def get_db_component(request):
     format_['data'] = [{'name': element, 'value': element} for element in data]
     format_['status'] = 0
   except Exception, e:
-    message = _('Error accessing the database %s: %s') % (name, e)
+    message = _('Error accessing the database: %s') % e
     LOG.warn(message)
     format_['message'] = message
 
   return JsonResponse(format_)
 
+def _get_db(request):
+  source = json.loads(request.POST.get('source', request.POST.get('fileFormat', '{}')))
+  user = User.objects.get(username=request.user)
+  name = None
+
+  if source['rdbmsMode'] == 'configRdbms':
+    if source['rdbmsType'] != 'jdbc':
+      query_server = rdbms.get_query_server_config(server=source['rdbmsType'])
+      db = rdbms.get(user, query_server=query_server)
+    else:
+      interpreters = get_ordered_interpreters(request.user)
+      options = {}
+      key = [key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName']]
+      if key:
+        options = key[0]['options']
+
+        db = Jdbc(driver_name=options['driver'], url=options['url'], username=options['user'], password=options['password'])
+  else:
+    name = source['rdbmsType']
+    if name != 'jdbc':
+      query_server = {
+        'server_name': name,
+        'server_host': source['rdbmsHostname'],
+        'server_port': int(source['rdbmsPort']),
+        'username': source['rdbmsUsername'],
+        'password': source['rdbmsPassword'],
+        'options': {},
+        'alias': name
+      }
+      db = rdbms.get(user, query_server=query_server)
+    else:
+      db = Jdbc(driver_name=source['rdbmsJdbcDriver'], url=source['rdbmsHostname'], username=source['rdbmsUsername'], password=source['rdbmsPassword'])
+
+  return db
+
 def jdbc_db_list(request):
   format_ = {'data': [], 'status': 1}
   interpreters = get_ordered_interpreters(request.user)
@@ -105,6 +113,7 @@ def get_drivers(request):
   servers_dict = dict(get_server_choices())
   format_['data'] = [{'value': key, 'name': servers_dict[key]} for key in servers_dict.keys()]
   format_['data'].append({'value': 'jdbc', 'name': 'JDBC'})
+#   format_['data'].append({'value': 'sqlalchemy', 'name': 'SQL Alchemy'})
   format_['status'] = 0
 
   return JsonResponse(format_)
@@ -254,21 +263,26 @@ def _splitby_column_check(statement, destination_splitby_column):
 
 class RdbmsIndexer():
 
-  def __init__(self, user, db_conf_name):
+  def __init__(self, user, db_conf_name, db=None):
     self.user = user
     self.db_conf_name = db_conf_name
+    self.db = db
 
   def guess_format(self):
     return {"type": "csv"}
 
   def get_sample_data(self, mode=None, database=None, table=None, column=None):
-    query_server = rdbms.get_query_server_config(server=self.db_conf_name)
-    db = rdbms.get(self.user, query_server=query_server)
+    if self.db:
+      db = self.db
+    else:
+      query_server = rdbms.get_query_server_config(server=self.db_conf_name)
+      db = rdbms.get(self.user, query_server=query_server)
 
-    if mode == 'configRdbms':
+    if mode == 'configRdbms' or self.db_conf_name != 'jdbc':
       assist = Assist(db)
     else:
       assist = JdbcAssist(db)
+
     response = {'status': -1}
     sample_data = assist.get_sample_data(database, table, column)
 

+ 4 - 2
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -246,8 +246,8 @@ ${ assist.assistPanel() }
               <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: createWizard.source.path, filechooser: createWizard.source.path, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true, openOnFocus: true, selectFolder: false }" placeholder="${ _('Click or drag from the assist') }">
             </label>
             <!-- ko if: createWizard.source.path().length > 0 -->
-              <a data-bind="storageContextPopover: { path: createWizard.source.path(), offset: { right: 5 } }" title="${ _('Open') }" style="font-size: 14px" class="margin-left-10">
-                <i class="fa fa-external-link-square"></i>
+              <a data-bind="storageContextPopover: { path: createWizard.source.path(), offset: { right: 5 } }" title="${ _('Preview') }" style="font-size: 14px" class="margin-left-10">
+                <i class="fa fa-fw fa-info"></i>
               </a>
             <!-- /ko -->
           </div>
@@ -1735,6 +1735,7 @@ ${ assist.assistPanel() }
         }
       });
       self.streamCheckConnection = function() {
+        $(".jHueNotify").remove();
         $.post("${ url('indexer:get_db_component') }", {
           "source": ko.mapping.toJSON(self)
         }, function (resp) {
@@ -2096,6 +2097,7 @@ ${ assist.assistPanel() }
         };
         self.sqoopJobLibPaths.push(newValue);
       };
+      self.addSqoopJobLibPath();
       self.removeSqoopJobLibPath = function (valueToRemove) {
         self.sqoopJobLibPaths.remove(valueToRemove);
       };