Forráskód Böngészése

HUE-5304 [indexer] Provide lib path to morphline indexer

Romain Rigaux 8 éve
szülő
commit
4b31b81

+ 3 - 12
desktop/libs/indexer/src/indexer/api3.py

@@ -149,15 +149,6 @@ def guess_field_types(request):
   return JsonResponse(format_)
 
 
-def index_file(request):
-  file_format = json.loads(request.POST.get('fileFormat', '{}'))
-  _convert_format(file_format["format"], inverse=True)
-  collection_name = file_format["name"]
-
-  job_handle = _index(request, file_format, collection_name)
-  return JsonResponse(job_handle)
-
-
 @api_error_handler
 def importer_submit(request):
   source = json.loads(request.POST.get('source', '{}'))
@@ -172,7 +163,7 @@ def importer_submit(request):
 
     if destination['indexerRunJob']:
       _convert_format(source["format"], inverse=True)
-      job_handle = _index(request, source, index_name, start_time=start_time)
+      job_handle = _index(request, source, index_name, start_time=start_time, lib_path=destination['indexerJobLibPath'])
     else:
       client = SolrClient(request.user)
       unique_key_field = destination['indexerDefaultField'] and destination['indexerDefaultField'][0] or None
@@ -247,7 +238,7 @@ def _create_table(request, source, destination, start_time=-1):
   return notebook.execute(request, batch=False)
 
 
-def _index(request, file_format, collection_name, query=None, start_time=None):
+def _index(request, file_format, collection_name, query=None, start_time=None, lib_path=None):
   indexer = MorphlineIndexer(request.user, request.fs)
 
   unique_field = indexer.get_unique_field(file_format)
@@ -282,4 +273,4 @@ def _index(request, file_format, collection_name, query=None, start_time=None):
 
   morphline = indexer.generate_morphline_config(collection_name, file_format, unique_field)
 
-  return indexer.run_morphline(request, collection_name, morphline, input_path, query, start_time=start_time)
+  return indexer.run_morphline(request, collection_name, morphline, input_path, query, start_time=start_time, lib_path=lib_path)

+ 2 - 2
desktop/libs/indexer/src/indexer/indexers/morphline.py

@@ -64,7 +64,7 @@ class MorphlineIndexer(object):
 
     return hdfs_workspace_path
 
-  def run_morphline(self, request, collection_name, morphline, input_path, query=None, start_time=None):
+  def run_morphline(self, request, collection_name, morphline, input_path, query=None, start_time=None, lib_path=None):
     workspace_path = self._upload_workspace(morphline)
 
     task = make_notebook(
@@ -92,7 +92,7 @@ class MorphlineIndexer(object):
 
     task.add_java_snippet(
       clazz='org.apache.solr.hadoop.MapReduceIndexerTool',
-      app_jar=CONFIG_INDEXER_LIBS_PATH.get(),
+      app_jar=lib_path if lib_path is not None else CONFIG_INDEXER_LIBS_PATH.get(),
       arguments=[
           u'--morphline-file',
           u'morphline.conf',

+ 5 - 3
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -20,7 +20,7 @@
   from desktop import conf
   from desktop.views import commonheader, commonfooter, commonshare, commonimportexport, _ko
 
-  from indexer.conf import ENABLE_NEW_INDEXER
+  from indexer.conf import ENABLE_NEW_INDEXER, CONFIG_INDEXER_LIBS_PATH
 %>
 
 <%namespace name="actionbar" file="actionbar.mako" />
@@ -1395,7 +1395,9 @@ ${ assist.assistPanel() }
           });
           $.post("/indexer/api/configs/list/", function (data) {
             if (data.status == 0) {
-              self.indexerConfigSets(data.configs);
+              self.indexerConfigSets(data.configs.map(function (config) {
+                return {'name': config, 'value': config};
+              }));
             } else {
               $(document).trigger("error", data.message);
             }
@@ -1545,7 +1547,7 @@ ${ assist.assistPanel() }
 
       // Index
       self.indexerRunJob = ko.observable(false);
-      self.indexerJobLibPath = ko.observable('');
+      self.indexerJobLibPath = ko.observable('${ CONFIG_INDEXER_LIBS_PATH.get() }');
       self.indexerConfigSet = ko.observable('');
       self.indexerConfigSets = ko.observableArray([]);
       self.indexerNumShards = ko.observable('');

+ 0 - 1
desktop/libs/indexer/src/indexer/urls.py

@@ -62,7 +62,6 @@ urlpatterns += patterns('indexer.solr_api',
 urlpatterns += patterns('indexer.api3',
   # Importer
   url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
-  url(r'^api/indexer/index_file/$', 'index_file', name='index_file'),
   url(r'^api/indexer/guess_field_types/$', 'guess_field_types', name='guess_field_types'),
 
   url(r'^api/importer/submit', 'importer_submit', name='importer_submit')