ソースを参照

HUE-5304 [indexer] Adding template variables to create a new collection

Romain Rigaux 8 年 前
コミット
a75b45b

+ 6 - 6
desktop/libs/indexer/src/indexer/api3.py

@@ -32,7 +32,7 @@ from notebook.models import make_notebook
 from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
-from indexer.indexers.morphline import MorhlineIndexer
+from indexer.indexers.morphline import MorphlineIndexer
 from indexer.indexers.sql import SQLIndexer
 from indexer.solr_client import SolrClient, SolrClientException
 
@@ -74,7 +74,7 @@ def guess_format(request):
   file_format = json.loads(request.POST.get('fileFormat', '{}'))
 
   if file_format['inputFormat'] == 'file':
-    indexer = MorhlineIndexer(request.user, request.fs)
+    indexer = MorphlineIndexer(request.user, request.fs)
     if not request.fs.isfile(file_format["path"]):
       raise PopupException(_('Path %(path)s is not a file') % file_format)
 
@@ -108,7 +108,7 @@ def guess_field_types(request):
   file_format = json.loads(request.POST.get('fileFormat', '{}'))
 
   if file_format['inputFormat'] == 'file':
-    indexer = MorhlineIndexer(request.user, request.fs)
+    indexer = MorphlineIndexer(request.user, request.fs)
     stream = request.fs.open(file_format["path"])
     _convert_format(file_format["format"], inverse=True)
 
@@ -172,14 +172,14 @@ def importer_submit(request):
     source['columns'] = destination['columns']
     job_handle = _index(request, source, collection_name, start_time=start_time)
   elif destination['ouputFormat'] == 'database':
-    job_handle = create_database(request, source, destination, start_time)
+    job_handle = _create_database(request, source, destination, start_time)
   else:
     job_handle = _create_table(request, source, destination, start_time)
 
   return JsonResponse(job_handle)
 
 
-def create_database(request, source, destination, start_time):
+def _create_database(request, source, destination, start_time):
   database = destination['name']
   comment = destination['description']
 
@@ -218,7 +218,7 @@ def _create_table(request, source, destination, start_time=-1):
 
 
 def _index(request, file_format, collection_name, query=None, start_time=None):
-  indexer = MorhlineIndexer(request.user, request.fs)
+  indexer = MorphlineIndexer(request.user, request.fs)
 
   unique_field = indexer.get_unique_field(file_format)
   is_unique_generated = indexer.is_unique_generated(file_format)

+ 2 - 2
desktop/libs/indexer/src/indexer/indexers/morphline.py

@@ -39,7 +39,7 @@ from indexer.indexers.morphline_operations import get_checked_args
 LOG = logging.getLogger(__name__)
 
 
-class MorhlineIndexer(object):
+class MorphlineIndexer(object):
 
   def __init__(self, username, fs=None, jt=None):
     self.fs = fs
@@ -68,7 +68,7 @@ class MorhlineIndexer(object):
     workspace_path = self._upload_workspace(morphline)
 
     task = make_notebook(
-      name=_('MorhlineIndexer job for %s') % collection_name,
+      name=_('MorphlineIndexer job for %s') % collection_name,
       editor_type='notebook',
       on_success_url=reverse('search:browse', kwargs={'name': collection_name}),
       is_task=True,

+ 7 - 7
desktop/libs/indexer/src/indexer/indexers/morphline_tests.py

@@ -31,14 +31,14 @@ from indexer.controller import CollectionManagerController
 from indexer.file_format import ApacheCombinedFormat, RubyLogFormat, HueLogFormat
 from indexer.fields import Field
 from indexer.indexers.morphline_operations import get_operator
-from indexer.indexers.morphline import MorhlineIndexer
+from indexer.indexers.morphline import MorphlineIndexer
 
 
 LOG = logging.getLogger(__name__)
 
 
 def _test_fixed_type_format_generate_morphline(format_):
-  indexer = MorhlineIndexer("test")
+  indexer = MorphlineIndexer("test")
   format_instance = format_()
 
   morphline = indexer.generate_morphline_config("test_collection", {
@@ -52,7 +52,7 @@ def _test_generate_field_operation_morphline(operation_format):
   fields = TestIndexer.simpleCSVFields[:]
   fields[0]['operations'].append(operation_format)
 
-  indexer = MorhlineIndexer("test")
+  indexer = MorphlineIndexer("test")
   morphline =indexer.generate_morphline_config("test_collection", {
       "columns": fields,
       "format": TestIndexer.simpleCSVFormat
@@ -125,7 +125,7 @@ class TestIndexer():
 
   def test_guess_csv_format(self):
     stream = StringIO.StringIO(TestIndexer.simpleCSVString)
-    indexer = MorhlineIndexer("test")
+    indexer = MorphlineIndexer("test")
 
     guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
 
@@ -143,7 +143,7 @@ class TestIndexer():
         assert_equal(expected[key], actual[key])
 
   def test_guess_format_invalid_csv_format(self):
-    indexer = MorhlineIndexer("test")
+    indexer = MorphlineIndexer("test")
     stream = StringIO.StringIO(TestIndexer.simpleCSVString)
 
     guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
@@ -170,7 +170,7 @@ class TestIndexer():
     assert_equal(fields, [])
 
   def test_generate_csv_morphline(self):
-    indexer = MorhlineIndexer("test")
+    indexer = MorphlineIndexer("test")
     morphline =indexer.generate_morphline_config("test_collection", {
         "columns": self.simpleCSVFields,
         "format": self.simpleCSVFormat
@@ -258,7 +258,7 @@ class TestIndexer():
     make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
     user = User.objects.get(username="test")
     collection_name = "test_collection"
-    indexer = MorhlineIndexer("test", fs=fs, jt=cluster.jt)
+    indexer = MorphlineIndexer("test", fs=fs, jt=cluster.jt)
     input_loc = "/tmp/test.csv"
 
     # upload the test file to hdfs

+ 77 - 3
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -656,6 +656,73 @@ ${ assist.assistPanel() }
         </div>
         <!-- /ko -->
 
+        <!-- ko if: outputFormat() == 'index' -->
+        <div class="card step">
+          <h3 class="card-heading simple">${_('Properties')}</h3>
+          <div class="card-body">
+            <div class="control-group">
+              <label class="checkbox inline-block">
+                <input type="checkbox" data-bind="checked: indexerRunJob"> ${_('Run Indexer job')}
+              </label>
+            </div>
+
+            <div class="control-group">
+              <label for="kuduPks" class="control-label"><div>${ _('Primary key') }</div>
+                <select id="kuduPks" data-bind="selectize: columns, selectedOptions: primaryKeys, selectedObjects: primaryKeyObjects, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="1" multiple="false"></select>
+              </label>
+            </div>
+
+            <div class="control-group">
+              <label for="kuduPks" class="control-label"><div>${ _('Default field') }</div>
+                <select id="kuduPks" data-bind="selectize: columns, selectedOptions: primaryKeys, selectedObjects: primaryKeyObjects, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="1" multiple="false"></select>
+              </label>
+            </div>
+
+            <div class="control-group">
+              <label class="control-label"><div>${ _('Extras') }</div>
+                <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, click: function() { showProperties(!showProperties()) }" title="${ _('Show extra properties') }">
+                  <i class="fa fa-sliders fa-padding-top"></i>
+                </a>
+              </label>
+            </div>
+
+            <span data-bind="visible: showProperties">
+              <div class="control-group">
+                <label for="destinationFormat" class="control-label"><div>${ _('Config set') }</div>
+                  <select id="destinationFormat" data-bind="selectize: indexerConfigSets, value: indexerConfigSet, optionsValue: 'value', optionsText: 'name'"></select>
+                </label>
+              </div>
+
+              <div class="control-group">
+                <label for="path" class="control-label" data-bind="indexerRunJob"><div>${ _('Libs') }</div>
+                  <input type="text" class="form-control path input-xxlarge" data-bind="value: indexerJobLibPath, filechooser: indexerJobLibPath, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
+                </label>
+              </div>
+
+              <div class="control-group">
+                <label for="indexerNumShards" class="control-label"><div>${ _('Num shards') }</div>
+                  <input type="number" class="input-small" placeholder="1" data-bind="value: indexerNumShards">
+                </label>
+              </div>
+              
+              <div class="control-group">
+                <label for="indexerReplicationFactor" class="control-label"><div>${ _('Replication factor') }</div>
+                  <input type="number" class="input-small" placeholder="1" data-bind="value: indexerReplicationFactor">
+                </label>
+              </div>
+
+              ## Router, maxShardsPer, shards, routerField
+
+              <div class="control-group" data-bind="visible: $root.createWizard.source.inputFormat() == 'file'">
+                <label class="checkbox inline-block">
+                  <input type="checkbox" data-bind="checked: hasHeader"> ${_('Use first row as header')}
+                </label>
+              </div>
+            </span>
+          </div>
+        </div>
+        <!-- /ko -->
+
         <!-- ko if: outputFormat() == 'table' || outputFormat() == 'index' -->
           <div class="card step">
             <h3 class="card-heading simple show-edit-on-hover">${_('Fields')} <!-- ko if: $root.createWizard.isGuessingFieldTypes --><i class="fa fa-spinner fa-spin"></i><!-- /ko --> <a class="inactive-action pointer" data-bind="visible: columns().length > 0" href="#fieldsBulkEditor" data-toggle="modal"><i class="fa fa-edit"></i></a></h3>
@@ -852,7 +919,7 @@ ${ assist.assistPanel() }
   </span>
 
   <!-- ko if: operations().length == 0 -->
-  <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
+  <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation, visible: $root.createWizard.destination.indexerRunJob" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
   <!-- /ko -->
 
   <div data-bind="foreach: operations">
@@ -876,7 +943,7 @@ ${ assist.assistPanel() }
     <!-- /ko -->
     <a class="pointer margin-left-20" data-bind="click: function(){$root.createWizard.removeOperation(operation, list)}" title="${ _('Remove') }"><i class="fa fa-times"></i></a>
     <div class="margin-left-20" data-bind="foreach: operation.fields">
-      <div data-bind="template: { name:'index-field-template', data:$data }" class="margin-top-10 field index-field"></div>
+      <div data-bind="template: { name: 'index-field-template', data: $data }" class="margin-top-10 field index-field"></div>
     </div>
   </div>
 </script>
@@ -1313,6 +1380,7 @@ ${ assist.assistPanel() }
           }, function (data) {
             self.isTargetExisting(data.status == 0);
             self.isTargetChecking(false);
+            // Fetch indexerConfigSets
           }).fail(function (xhr, textStatus, errorThrown) { self.isTargetExisting(false); self.isTargetChecking(false); });
         }
         resizeElements();
@@ -1331,7 +1399,7 @@ ${ assist.assistPanel() }
       self.outputFormatsList = ko.observableArray([
           {'name': 'Table', 'value': 'table'},
           % if ENABLE_NEW_INDEXER.get():
-          {'name': 'Solr index', 'value': 'index'},
+          {'name': 'Search index', 'value': 'index'},
           % endif
           {'name': 'File', 'value': 'file'},
           {'name': 'Database', 'value': 'database'},
@@ -1456,6 +1524,12 @@ ${ assist.assistPanel() }
       self.customRegexp = ko.observable('');
 
       // Index
+      self.indexerRunJob = ko.observable(false);
+      self.indexerJobLibPath = ko.observable('');
+      self.indexerConfigSet = ko.observable('');
+      self.indexerConfigSets = ko.observableArray([]);
+      self.indexerNumShards = ko.observable('');
+      self.indexerReplicationFactor = ko.observable(1);
     };
 
     var CreateWizard = function (vm) {