Эх сурвалжийг харах

HUE-6659 [import] Cleaned code based on the code review. Added rdbms.py. Replaced test connection checkbox with button

Prachi Poddar 8 жил өмнө
parent
commit
f605eef99d

+ 1 - 1
apps/oozie/src/oozie/templates/editor2/gen/workflow-sqoop.xml.mako

@@ -41,4 +41,4 @@
         <ok to="${ node_mapping[node['children'][0]['to']].name }"/>
         <ok to="${ node_mapping[node['children'][0]['to']].name }"/>
         <error to="${ node_mapping[node['children'][1]['error']].name }"/>
         <error to="${ node_mapping[node['children'][1]['error']].name }"/>
         ${ common.sla(node) }
         ${ common.sla(node) }
-    </action>
+    </action>

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1257,6 +1257,9 @@
   # Flag to turn on the new metadata importer.
   # Flag to turn on the new metadata importer.
   ## enable_new_importer=false
   ## enable_new_importer=false
 
 
+  # Flag to turn on sqoop.
+  ## enable_sqoop=false
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure Job Designer
 # Settings to configure Job Designer

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1259,6 +1259,9 @@
   # Flag to turn on the new metadata importer.
   # Flag to turn on the new metadata importer.
   ## enable_new_importer=false
   ## enable_new_importer=false
 
 
+  # Flag to turn on sqoop.
+  ## enable_sqoop=false
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure Job Designer
 # Settings to configure Job Designer

+ 9 - 166
desktop/libs/indexer/src/indexer/api3.py

@@ -18,7 +18,6 @@
 import json
 import json
 import logging
 import logging
 
 
-from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
@@ -26,11 +25,8 @@ from desktop.lib import django_mako
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document2
 from desktop.models import Document2
-from hadoop import conf
 from librdbms.server import dbms as rdbms
 from librdbms.server import dbms as rdbms
-from librdbms.conf import DATABASES, get_database_password
 from notebook.connectors.base import get_api, Notebook
 from notebook.connectors.base import get_api, Notebook
-from notebook.connectors.rdbms import Assist
 from notebook.decorators import api_error_handler
 from notebook.decorators import api_error_handler
 from notebook.models import make_notebook
 from notebook.models import make_notebook
 
 
@@ -38,6 +34,7 @@ from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
 from indexer.fields import Field
 from indexer.indexers.morphline import MorphlineIndexer
 from indexer.indexers.morphline import MorphlineIndexer
+from indexer.indexers.rdbms import RdbmsIndexer, run_sqoop
 from indexer.indexers.sql import SQLIndexer
 from indexer.indexers.sql import SQLIndexer
 from indexer.solr_client import SolrClient, MAX_UPLOAD_SIZE
 from indexer.solr_client import SolrClient, MAX_UPLOAD_SIZE
 
 
@@ -105,7 +102,7 @@ def guess_format(request):
   elif file_format['inputFormat'] == 'query':
   elif file_format['inputFormat'] == 'query':
     format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\u0001"}
     format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\u0001"}
   elif file_format['inputFormat'] == 'rdbms':
   elif file_format['inputFormat'] == 'rdbms':
-    format_ = {"type": "csv"}
+    format_ = RdbmsIndexer(request.user, file_format['rdbmsType']).guess_format()
 
 
   format_['status'] = 0
   format_['status'] = 0
   return JsonResponse(format_)
   return JsonResponse(format_)
@@ -155,12 +152,11 @@ def guess_field_types(request):
   elif file_format['inputFormat'] == 'rdbms':
   elif file_format['inputFormat'] == 'rdbms':
     query_server = rdbms.get_query_server_config(server=file_format['rdbmsType'])
     query_server = rdbms.get_query_server_config(server=file_format['rdbmsType'])
     db = rdbms.get(request.user, query_server=query_server)
     db = rdbms.get(request.user, query_server=query_server)
-    assist = Assist(db)
-    sample = assist.get_sample_data(database=file_format['rdbmsDatabaseName'], table=file_format['rdbmsTableName'])
+    sample = RdbmsIndexer(request.user, file_format['rdbmsType']).get_sample_data(database=file_format['rdbmsDatabaseName'], table=file_format['rdbmsTableName'])
     table_metadata = db.get_columns(file_format['rdbmsDatabaseName'], file_format['rdbmsTableName'], names_only=False)
     table_metadata = db.get_columns(file_format['rdbmsDatabaseName'], file_format['rdbmsTableName'], names_only=False)
 
 
     format_ = {
     format_ = {
-        "sample": list(sample.rows())[:4],
+        "sample": list(sample['rows'])[:4],
         "columns": [
         "columns": [
             Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
             Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
             for col in table_metadata
             for col in table_metadata
@@ -170,93 +166,6 @@ def guess_field_types(request):
   return JsonResponse(format_)
   return JsonResponse(format_)
 
 
 
 
-def get_databases(request):
-  source = json.loads(request.POST.get('source', '{}'))
-  user = User.objects.get(username=request.user)
-  if source['rdbmsMode'] == 'configRdbms':
-    query_server = rdbms.get_query_server_config(server=source['rdbmsType'])
-  else:
-    name = source['rdbmsType']
-    if name:
-      query_server = {
-        'server_name': str(name),
-        'server_host': str(source['rdbmsHostname']),
-        'server_port': int(source['rdbmsPort']),
-        'username': str(source['rdbmsUsername']),
-        'password': str(source['rdbmsPassword']),
-        'options': {},
-        'alias': name
-      }
-    LOG.debug("Query Server: %s" % query_server)
-
-  db = rdbms.get(user, query_server=query_server)
-  assist = Assist(db)
-  data = assist.get_databases() #format of data ['abc','def','ghi',...,'xyz']
-  format_ = {}
-  if data:
-    list = []
-    for element in data:
-      dict = {}
-      dict['name'] = element
-      dict['value'] = element
-      list.append(dict)
-    format_['data'] = list
-    format_['status'] = 0
-  else:
-    format_ = {}
-    format_['data'] = []
-    format_['status'] = 1
-  print format_
-  return JsonResponse(format_)
-
-
-def get_tables(request):
-  source = json.loads(request.POST.get('source', '{}'))
-  user = User.objects.get(username=request.user)
-  if source['rdbmsMode'] == 'configRdbms':
-    query_server = rdbms.get_query_server_config(server=source['rdbmsType'])
-  else:
-    name = source['rdbmsType']
-    if name:
-      query_server = {
-        'server_name': str(name),
-        'server_host': str(source['rdbmsHostname']),
-        'server_port': int(source['rdbmsPort']),
-        'username': str(source['rdbmsUsername']),
-        'password': str(source['rdbmsPassword']),
-        'options': {},
-        'alias': name
-      }
-    LOG.debug("Query Server: %s" % query_server)
-
-  db = rdbms.get(user, query_server=query_server)
-  assist = Assist(db)
-  data = assist.get_tables(source['rdbmsDatabaseName']) ##format of data ['abc','def','ghi',...,'xyz']
-  format_ = {}
-  if data:
-    list = []
-    for element in data:
-      dict = {}
-      dict['name'] = element
-      dict['value'] = element
-      list.append(dict)
-    format_['data'] = list
-    format_['status'] = 0
-  else:
-    format_ = []
-  print format_
-  return JsonResponse(format_)
-
-
-def index_file(request):
-  file_format = json.loads(request.POST.get('fileFormat', '{}'))
-  _convert_format(file_format["format"], inverse=True)
-  collection_name = file_format["name"]
-
-  job_handle = _index(request, file_format, collection_name)
-  return JsonResponse(job_handle)
-
-
 @api_error_handler
 @api_error_handler
 def importer_submit(request):
 def importer_submit(request):
   source = json.loads(request.POST.get('source', '{}'))
   source = json.loads(request.POST.get('source', '{}'))
@@ -277,15 +186,10 @@ def importer_submit(request):
       job_handle = _create_index(request.user, request.fs, client, source, destination, index_name)
       job_handle = _create_index(request.user, request.fs, client, source, destination, index_name)
   elif destination['ouputFormat'] == 'database':
   elif destination['ouputFormat'] == 'database':
     job_handle = _create_database(request, source, destination, start_time)
     job_handle = _create_database(request, source, destination, start_time)
-  elif destination['outputFormat'] == 'file' and source['inputFormat'] == 'rdbms':
-    job_handle = run_sqoop(request, source, destination, start_time)
-  elif destination['outputFormat'] == 'hive' and source['inputFormat'] == 'rdbms':
-    job_handle = run_sqoop(request, source, destination, start_time)
-  elif destination['outputFormat'] == 'hbase' and source['inputFormat'] == 'rdbms':
-    job_handle = run_sqoop(request, source, destination, start_time)
-  else:
-    job_handle = _create_table(request, source, destination, start_time)
-  print JsonResponse(job_handle)
+  elif source['inputFormat'] == 'rdbms':
+    if destination['outputFormat'] in ('file', 'table', 'hbase'):
+      job_handle = run_sqoop(request, source, destination, start_time)
+
   return JsonResponse(job_handle)
   return JsonResponse(job_handle)
 
 
 
 
@@ -391,6 +295,7 @@ def _index(request, file_format, collection_name, query=None, start_time=None, l
       fields=request.POST.get('fields', schema_fields),
       fields=request.POST.get('fields', schema_fields),
       unique_key_field=unique_field
       unique_key_field=unique_field
     )
     )
+
   if file_format['inputFormat'] == 'table':
   if file_format['inputFormat'] == 'table':
     db = dbms.get(request.user)
     db = dbms.get(request.user)
     table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
     table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
@@ -407,65 +312,3 @@ def _index(request, file_format, collection_name, query=None, start_time=None, l
   morphline = indexer.generate_morphline_config(collection_name, file_format, unique_field)
   morphline = indexer.generate_morphline_config(collection_name, file_format, unique_field)
 
 
   return indexer.run_morphline(request, collection_name, morphline, input_path, query, start_time=start_time, lib_path=lib_path)
   return indexer.run_morphline(request, collection_name, morphline, input_path, query, start_time=start_time, lib_path=lib_path)
-
-def run_sqoop(request, source, destination, start_time):
-  rdbmsMode = str(source['rdbmsMode'])
-  rdbmsName = str(source['rdbmsType'])
-  rdbmsDatabaseName = str(source['rdbmsDatabaseName'])
-  allTablesSelected = str(source['allTablesSelected'])
-  destinationType = str(destination['outputFormat'])
-
-  if not allTablesSelected:
-    rdbmsTableName = str(source['rdbmsTableName'])
-
-  if rdbmsMode == 'configRdbms':
-    rdbmsHost = str(DATABASES[rdbmsName].HOST.get())
-    rdbmsPort = str(DATABASES[rdbmsName].PORT.get())
-    rdbmsUserName = str(DATABASES[rdbmsName].USER.get())
-    rdbmsPassword = str(get_database_password(rdbmsName))
-  else:
-    rdbmsHost = str(source['rdbmsHostname'])
-    rdbmsPort = str(source['rdbmsPort'])
-    rdbmsUserName = str(source['rdbmsUsername'])
-    rdbmsPassword = str(source['rdbmsPassword'])
-
-  if destinationType == 'file':
-    targetDir = conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.get()+str(destination['name'])+'/test'
-
-  #print rdbmsName
-  #print rdbmsHost
-  #print rdbmsPort
-  #print rdbmsDatabaseName
-  #print rdbmsUserName
-  #print rdbmsPassword
-  #print targetDir
-  #print 'import --connect jdbc:'+rdbmsName+'://'+'127.0.0.1'+':'+str(rdbmsPort)+'/'+rdbmsDatabaseName+' --username '+rdbmsUserName+' --password '+rdbmsPassword+' --query \'SELECT * FROM '+rdbmsTableName+' as a WHERE $CONDITIONS\' --target-dir '+targetDir+' --verbose --split-by a.empid'
-
-  if destinationType == 'file':
-    if allTablesSelected:
-      statement='import-all-tables --connect jdbc:'+rdbmsName+'://'+rdbmsHost+':'+rdbmsPort+'/'+rdbmsDatabaseName+' --username '+rdbmsUserName+' --password '+rdbmsPassword+' --warehouse-dir '+targetDir+' -m 1'
-    else:
-      statement = 'import --connect jdbc:'+rdbmsName+'://'+rdbmsHost+':'+rdbmsPort+'/'+rdbmsDatabaseName+' --username '+rdbmsUserName+' --password '+rdbmsPassword+' --table '+rdbmsTableName+' --target-dir '+ targetDir+' -m 1'
-  elif destinationType == 'hive':
-    if allTablesSelected:
-      statement = 'import-all-tables --connect jdbc:'+rdbmsName+'://'+rdbmsHost+':'+rdbmsPort+'/'+rdbmsDatabaseName+' --username '+rdbmsUserName+' --password '+rdbmsPassword+' --hive-import'
-    else:
-      statement = 'import --connect jdbc:'+rdbmsName+'://'+rdbmsHost+':'+rdbmsPort+'/'+rdbmsDatabaseName+' --username '+rdbmsUserName+' --password '+rdbmsPassword+' --table '+rdbmsTableName+' --hive-import'
-
-  print statement
-  task = make_notebook(
-      name=_('Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s') % {
-          'rdbmsDatabaseName': source['rdbmsDatabaseName'],
-          'rdbmsDatabaseName': source['rdbmsDatabaseName'],
-          'path': destination['name']
-        },
-      editor_type='sqoop1',
-      statement=statement,
-      files = [{"path": "/user/admin/mysql-connector-java.jar", "type": "jar"}],
-      status='ready',
-      on_success_url='/filebrowser/view/%s(name)s' % destination,
-      last_executed=start_time,
-      is_task=True
-  )
-
-  return task.execute(request, batch=True)

+ 179 - 0
desktop/libs/indexer/src/indexer/indexers/rdbms.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
+from django.utils.translation import ugettext as _
+
+from desktop.lib.django_util import JsonResponse
+from librdbms.server import dbms as rdbms
+from librdbms.conf import DATABASES, get_database_password
+from notebook.connectors.rdbms import Assist
+from notebook.models import make_notebook
+
+
+LOG = logging.getLogger(__name__)
+
+
+def get_db_component(request):
+  format_ = {'data': [], 'status': 1, 'message': ''}
+  name = None
+  try:
+    source = json.loads(request.POST.get('source', '{}'))
+    user = User.objects.get(username=request.user)
+    if source['rdbmsMode'] == 'configRdbms':
+      query_server = rdbms.get_query_server_config(server=source['rdbmsType'])
+    else:
+      name = source['rdbmsType']
+      if name:
+        query_server = {
+          'server_name': name,
+          'server_host': source['rdbmsHostname'],
+          'server_port': int(source['rdbmsPort']),
+          'username': source['rdbmsUsername'],
+          'password': source['rdbmsPassword'],
+          'options': {},
+          'alias': name
+        }
+
+    db = rdbms.get(user, query_server=query_server)
+    assist = Assist(db)
+    if not source['rdbmsDatabaseName']:
+      data = assist.get_databases()
+    else:
+      data = assist.get_tables(source['rdbmsDatabaseName'])
+    format_['data'] = [{'name': element, 'value': element} for element in data]
+    format_['status'] = 0
+  except Exception, e:
+    message = 'Error accessing the database %s' % name
+    LOG.warn(message, e)
+    format['message'] = _(message)
+
+  return JsonResponse(format_)
+
+
+def run_sqoop(request, source, destination, start_time):
+  rdbms_mode = source['rdbmsMode']
+  rdbms_name = source['rdbmsType']
+  rdbms_database_name = source['rdbmsDatabaseName']
+  rdbms_all_tables_selected = source['rdbmsAllTablesSelected']
+  destination_type = destination['outputFormat']
+  destination_name = destination['name']
+  destination_table_name = destination['tableName']
+  destination_database_name = destination['databaseName']
+
+  if not rdbms_all_tables_selected:
+    rdbms_table_name = source['rdbmsTableName']
+
+  if rdbms_mode == 'configRdbms':
+    rdbms_host = DATABASES[rdbms_name].HOST.get()
+    rdbms_port = DATABASES[rdbms_name].PORT.get()
+    rdbms_user_name = DATABASES[rdbms_name].USER.get()
+    rdbms_password = get_database_password(rdbms_name)
+  else:
+    rdbms_host = source['rdbmsHostname']
+    rdbms_port = source['rdbmsPort']
+    rdbms_user_name = source['rdbmsUsername']
+    rdbms_password = source['rdbmsPassword']
+
+  statement = '--connect jdbc:%(rdbmsName)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password %(rdbmsPassword)s' % {
+    'rdbmsName': rdbms_name,
+    'rdbmsHost': rdbms_host,
+    'rdbmsPort': rdbms_port,
+    'rdbmsDatabaseName': rdbms_database_name,
+    'rdbmsUserName': rdbms_user_name,
+    'rdbmsPassword': rdbms_password
+  }
+  if destination_type == 'file':
+    success_url = '/filebrowser/view/' + destination_name
+    targetDir = request.fs.fs_defaultfs + destination_name
+    if rdbms_all_tables_selected:
+      statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s -m 1' % {
+        'statement': statement,
+        'targetDir': targetDir
+      }
+    else:
+      statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s -m 1' % {
+        'statement': statement,
+        'rdbmsTableName': rdbms_table_name,
+        'targetDir': targetDir
+      }
+  elif destination_type == 'table':
+    success_url = reverse('metastore:describe_table', kwargs={'database': destination_database_name, 'table': destination_table_name})
+    if rdbms_all_tables_selected:
+      statement = 'import-all-tables %(statement)s --hive-import' % {
+        'statement': statement
+      }
+    else:
+      statement = 'import %(statement)s --table %(rdbmsTableName)s --hive-import' % {
+        'statement': statement,
+        'rdbmsTableName': rdbms_table_name
+      }
+  elif destination_type == 'hbase':
+    success_url = '/hbase/#HBase/' + destination_table_name
+    # Todo
+
+  lib_files = []
+  if destination['sqoopJobLibPath']:
+    lib_files.append({"path": destination['sqoopJobLibPath'], "type": "jar"})
+
+  task = make_notebook(
+    name=_('Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s') % {
+      'rdbmsDatabaseName': rdbms_database_name,
+      'rdbmsDatabaseName': rdbms_database_name,
+      'path': destination_name
+    },
+    editor_type='sqoop1',
+    statement=statement,
+    files=lib_files,
+    status='ready',
+    on_success_url=success_url,
+    last_executed=start_time,
+    is_task=True
+  )
+
+  return task.execute(request, batch=False)
+
+
+class RdbmsIndexer():
+
+  def __init__(self, user, db_conf_name):
+    self.user = user
+    self.db_conf_name = db_conf_name
+
+  def guess_format(self):
+    return {"type": "csv"}
+
+  def get_sample_data(self, database=None, table=None, column=None):
+    query_server = rdbms.get_query_server_config(server=self.db_conf_name)
+    db = rdbms.get(self.user, query_server=query_server)
+
+    assist = Assist(db)
+    response = {'status': -1}
+    sample_data = assist.get_sample_data(database, table, column)
+
+    if sample_data:
+      response['status'] = 0
+      response['headers'] = sample_data.columns
+      response['rows'] = list(sample_data.rows())
+    else:
+      response['message'] = _('Failed to get sample data.')
+
+    return response

+ 0 - 99
desktop/libs/indexer/src/indexer/rdbms_indexer.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.import logging
-
-import logging
-
-from librdbms.server import dbms
-from notebook.connectors.rdbms import Assist
-
-
-LOG = logging.getLogger(__name__)
-
-
-class RdbmsIndexer():
-
-  def __init__(self, user, db_conf_name):
-    self.user = user
-    self.db_conf_name = db_conf_name
-
-  def guess_type(self):
-    return {}
-
-  def guess_format(self):
-    return {}
-
-  def get_sample_data(self, database=None, table=None, column=None):
-    query_server = dbms.get_query_server_config(server=self.db_conf_name)
-    db = dbms.get(self.user, query_server=query_server)
-
-    assist = Assist(db)
-    response = {'status': -1}
-    sample_data = assist.get_sample_data(database, table, column)
-
-    if sample_data:
-      response['status'] = 0
-      response['headers'] = sample_data.columns
-      response['rows'] = list(sample_data.rows())
-    else:
-      response['message'] = _('Failed to get sample data.')
-
-    return response
-
-  def get_databases(self):
-    query_server = dbms.get_query_server_config(server=self.db_conf_name)
-    db = dbms.get(self.user, query_server=query_server)
-    assist = Assist(db)
-    response = {'status': -1}
-    sample_data = assist.get_databases()
-
-    if sample_data:
-      response['status'] = 0
-      response['data'] = sample_data
-    else:
-      response['message'] = _('Failed to get sample data.')
-
-    return response
-
-  def get_tables(self, database=None):
-    query_server = dbms.get_query_server_config(server=self.db_conf_name)
-    db = dbms.get(self.user, query_server=query_server)
-    assist = Assist(db)
-    response = {'status': -1}
-    sample_data = assist.get_tables(database)
-
-    if sample_data:
-      response['status'] = 0
-      response['data'] = sample_data
-    else:
-      response['message'] = _('Failed to get sample data.')
-
-    return response
-
-  def get_columns(self, database=None, table=None):
-    query_server = dbms.get_query_server_config(server=self.db_conf_name)
-    db = dbms.get(self.user, query_server=query_server)
-
-    assist = Assist(db)
-    response = {'status': -1}
-    sample_data = assist.get_columns(database, table)
-
-    if sample_data:
-      response = sample_data
-    else:
-      response['message'] = _('Failed to get sample data.')
-
-    return response

+ 27 - 41
desktop/libs/indexer/src/indexer/rdbms_indexer_tests.py

@@ -16,62 +16,48 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-import json
 import logging
 import logging
 
 
 from django.contrib.auth.models import User
 from django.contrib.auth.models import User
 
 
 from nose.plugins.skip import SkipTest
 from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal, assert_false, assert_not_equal, assert_true
 from nose.tools import assert_equal, assert_false, assert_not_equal, assert_true
+
+from desktop.auth.backend import rewrite_user
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
-from indexer.conf import ENABLE_SQOOP
 
 
-from indexer.rdbms_indexer import RdbmsIndexer
+from indexer.conf import ENABLE_SQOOP
+from indexer.indexers.rdbms import RdbmsIndexer
+from librdbms.server import dbms as rdbms
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
-
 class TestRdbmsIndexer():
 class TestRdbmsIndexer():
-  if not ENABLE_SQOOP.get():
-    raise SkipTest
-  '''
-  def test_get_databases(self):
-    self.client = make_logged_in_client()
-    self.user = User.objects.get(username='test')
-
-    indexer = RdbmsIndexer(self.user, db_conf_name='mysql')
-
-    data = indexer.get_databases()
-    assert_equal(1, data['status'], data)
 
 
-  '''
-  def test_get_sample_data(self):
-    self.client = make_logged_in_client()
-    self.user = User.objects.get(username='test')
-
-    indexer = RdbmsIndexer(self.user, db_conf_name='mysql')
-    data = indexer.get_sample_data(database='hue', table='employee', column='empname')
+  @classmethod
+  def setup_class(cls):
+    if not ENABLE_SQOOP.get():
+      raise SkipTest
+    if not rdbms.get_query_server_config(server='mysql'):
+      raise SkipTest
+    cls.client = make_logged_in_client()
+    cls.user = User.objects.get(username='test')
+    cls.user = rewrite_user(cls.user)
+    cls.indexer = RdbmsIndexer(cls.user, db_conf_name='mysql')
+
+  @classmethod
+  def teardown_class(cls):
+    cls.user.is_superuser = False
+    cls.user.save()
+
+  def test_get_sample_data(cls):
+    data = cls.indexer.get_sample_data(database='hue', table='employee', column='empname')
 
 
     assert_equal(0, data['status'], data)
     assert_equal(0, data['status'], data)
-    assert_equal('',data['rows'], data)
-  '''
-  def test_columns(self):
-    self.client = make_logged_in_client()
-    self.user = User.objects.get(username='test')
-
-    indexer = RdbmsIndexer(self.user, db_conf_name='mysql')
-    data = indexer.get_columns(database='hue', table='employee')
-
-    assert_not_equal([], data, data)
-    assert_true(data, data)
+    assert_not_equal('', data['rows'], data)
 
 
-  def test_get_tables(self):
-    self.client = make_logged_in_client()
-    self.user = User.objects.get(username='test')
+  def test_guess_format(cls):
+    data = cls.indexer.guess_format()
 
 
-    indexer = RdbmsIndexer(self.user, db_conf_name='mysql')
-
-    data = indexer.get_tables('hue')
-    assert_equal(0, data['status'], data)
-  '''
+    assert_equal({"type": "csv"}, data)

Файлын зөрүү хэтэрхий том тул дарагдсан байна
+ 179 - 264
desktop/libs/indexer/src/indexer/templates/importer.mako


+ 6 - 3
desktop/libs/indexer/src/indexer/urls.py

@@ -55,12 +55,15 @@ urlpatterns += patterns('indexer.api3',
   # Importer
   # Importer
   url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
   url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
   url(r'^api/indexer/guess_field_types/$', 'guess_field_types', name='guess_field_types'),
   url(r'^api/indexer/guess_field_types/$', 'guess_field_types', name='guess_field_types'),
-  url(r'^api/indexer/get_databases/$', 'get_databases', name='get_databases'),
-  url(r'^api/indexer/get_tables/$', 'get_tables', name='get_tables'),
 
 
   url(r'^api/importer/submit', 'importer_submit', name='importer_submit')
   url(r'^api/importer/submit', 'importer_submit', name='importer_submit')
 )
 )
 
 
+urlpatterns += patterns('indexer.indexers.rdbms',
+  url(r'^api/indexer/indexers/get_db_component/$', 'get_db_component', name='get_db_component')
+)
+
+
 # Deprecated
 # Deprecated
 urlpatterns += patterns('indexer.api',
 urlpatterns += patterns('indexer.api',
   url(r'^api/fields/parse/$', 'parse_fields', name='api_parse_fields'),
   url(r'^api/fields/parse/$', 'parse_fields', name='api_parse_fields'),
@@ -72,4 +75,4 @@ urlpatterns += patterns('indexer.api',
   url(r'^api/collections/(?P<collection>[^/]+)/fields/$', 'collections_fields', name='api_collections_fields'),
   url(r'^api/collections/(?P<collection>[^/]+)/fields/$', 'collections_fields', name='api_collections_fields'),
   url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
   url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
   url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
   url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
-)
+)

+ 1 - 1
desktop/libs/librdbms/src/librdbms/server/oracle_lib.py

@@ -111,4 +111,4 @@ class OracleClient(BaseRDMSClient):
   def get_sample_data(self, database, table, column=None, limit=100):
   def get_sample_data(self, database, table, column=None, limit=100):
     column = '"%s"' % column  if column else '*'
     column = '"%s"' % column  if column else '*'
     statement = 'SELECT %s FROM "%s"."%s" LIMIT %d' % (column, database, table, limit)
     statement = 'SELECT %s FROM "%s"."%s" LIMIT %d' % (column, database, table, limit)
-    return self.execute_statement(statement)
+    return self.execute_statement(statement)

Энэ ялгаанд хэт олон файл өөрчлөгдсөн тул зарим файлыг харуулаагүй болно