Sfoglia il codice sorgente

HUE-1450 [importer] Skeleton of refactored API with index()

Romain 5 anni fa
parent
commit
4264db764f

+ 27 - 1
desktop/libs/indexer/src/indexer/api3.py

@@ -53,6 +53,7 @@ from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
 from indexer.indexers.envelope import EnvelopeIndexer
+from indexer.indexers.base import get_api
 from indexer.indexers.flink_sql import FlinkIndexer
 from indexer.indexers.morphline import MorphlineIndexer
 from indexer.indexers.rdbms import run_sqoop, _get_api
@@ -430,7 +431,7 @@ def importer_submit(request):
       request.user,
       request.fs
     )
-    
+
     job_handle = api.create_table_from_kafka(**args)
 
     if request.POST.get('show_command'):
@@ -478,6 +479,31 @@ def importer_submit(request):
   return JsonResponse(job_handle)
 
 
+@require_POST
+@api_error_handler
+def index(request):
+  '''
+  Input: pasted data, CSV/json files, Kafka topic
+  Output: tables
+  '''
+  response = {'status': -1}
+
+  source = json.loads(request.POST.get('source', '{}'))
+  destination = json.loads(request.POST.get('destination', '{}'))
+  options = json.loads(request.POST.get('options', '{}'))
+  connector_id = request.POST.get('connector')
+
+  api = get_api(request.user, connector_id)
+
+  if request.FILES.get('data'):
+    data = request.FILES['data'].read()
+    print(data)
+
+  result = api.index(source, destination, options)
+
+  return JsonResponse({'result': result})
+
+
 def _small_indexing(user, fs, client, source, destination, index_name):
   kwargs = {}
   errors = []

+ 59 - 0
desktop/libs/indexer/src/indexer/indexers/base.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.utils.translation import ugettext as _
+
+from desktop.conf import has_connectors
+from desktop.lib.connectors.models import _get_installed_connectors
+from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.i18n import smart_unicode
+
+
+def get_api(user, connector_id):
+  if has_connectors() and connector_id != 'dummy':
+    connectors = _get_installed_connectors(user=user, connector_id=int(connector_id))
+    connector = connectors[0]
+    dialect = connector['dialect']
+  else:
+    connector = None  # Could get the interpreter if Connectors are off
+    dialect = connector_id
+
+  if dialect == 'dummy':
+    return Base(user, connector_id)
+  else:
+    raise PopupException(_('Indexer connector dialect not recognized: %s') % dialect)
+
+
+class Base():
+
+  def __init__(self, user, connector_id):
+    self.user = user
+    self.connector_id = connector_id
+
+  def index(self, source, destination, options=None): pass
+
+
+
+class IndexerApiException(Exception):
+  def __init__(self, message=None):
+    self.message = message or _('No error message, please check the logs.')
+
+  def __str__(self):
+    return str(self.message)
+
+  def __unicode__(self):
+    return smart_unicode(self.message)

+ 1 - 0
desktop/libs/indexer/src/indexer/urls.py

@@ -71,6 +71,7 @@ urlpatterns += [
   # Importer
   url(r'^api/indexer/guess_format/?$', indexer_api3.guess_format, name='guess_format'),
   url(r'^api/indexer/guess_field_types/?$', indexer_api3.guess_field_types, name='guess_field_types'),
+  url(r'^api/indexer/index/?$', indexer_api3.index, name='index'),
 
   url(r'^api/importer/submit', indexer_api3.importer_submit, name='importer_submit'),
   url(r'^api/importer/save/?$', indexer_api3.save_pipeline, name='save_pipeline'),