Просмотр исходного кода

HUE-8021 [indexer] Support splitting a field with SOLR REST indexing

jdesjean 7 лет назад
Родитель
Сommit
dd6b3cd

+ 7 - 0
desktop/libs/indexer/src/indexer/api3.py

@@ -248,6 +248,13 @@ def _small_indexing(user, fs, client, source, destination, index_name):
   skip_fields = [field['name'] for field in fields if not field['keep']]
 
   kwargs['fieldnames'] = ','.join([field['name'] for field in fields])
+  for field in fields:
+    for operation in field['operations']:
+      if operation['type'] == 'split':
+        field['multiValued'] = True # Solr requires multiValued to be set when splitting
+        kwargs['f.%(name)s.split' % field] = 'true'
+        kwargs['f.%(name)s.separator' % field] = operation['settings']['splitChar'] or ','
+
   if skip_fields:
     kwargs['skip'] = ','.join(skip_fields)
     fields = [field for field in fields if field['name'] not in skip_fields]

+ 2 - 1
desktop/libs/indexer/src/indexer/solr_client.py

@@ -121,7 +121,8 @@ class SolrClient(object):
         fields = [{
             'name': field['name'],
             'type': SolrClient._port_field_types(field)['type'],
-            'stored': field.get('stored', True)
+            'stored': field.get('stored', True),
+            'multiValued': field.get('multiValued', False)
           } for field in fields if field['name'] != 'id'
         ]
         self.api.add_fields(name, fields)

+ 15 - 2
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -911,7 +911,7 @@ ${ assist.assistPanel() }
   </span>
 
   <!-- ko if: operations().length == 0 -->
-  <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation, visible: $root.createWizard.destination.indexerRunJob" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
+  <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
   <!-- /ko -->
 
   <div class="inline-block" data-bind="template: { name: 'field-column-example' }"></div>
@@ -944,7 +944,7 @@ ${ assist.assistPanel() }
 
 <script type="text/html" id="operation-template">
   <div class="operation">
-    <select data-bind="browserAwareSelectize: $root.createWizard.operationTypes.map(function(o){return o.name}), value: operation.type"></select>
+    <select data-bind="browserAwareSelectize: $root.createWizard.operationTypesFiltered, value: operation.type"></select>
     <!-- ko template: "args-template" --><!-- /ko -->
     <!-- ko if: operation.settings().outputType() == "custom_fields" -->
       <label class="margin-left-5">${ _('Number of expected fields') }
@@ -1935,6 +1935,19 @@ ${ assist.assistPanel() }
       self.isIndexing = ko.observable(false);
       self.indexingError = ko.observable(false);
       self.indexingSuccess = ko.observable(false);
+      self.destination.indexerRunJob.subscribe(function () {
+        self.destination.columns().forEach(function (column) {
+          column.operations.removeAll();
+        });
+      });
+      self.operationTypesFiltered = ko.pureComputed(function () {
+        var indexerRunJob = self.destination.indexerRunJob();
+        return self.operationTypes.filter(function (operation) {
+          return indexerRunJob || operation.name === 'split'; // We only support split for now with CSV API
+        }).map(function (o) {
+          return o.name;
+        });
+      });
       self.indexFile = function () {
         if (!self.readyToIndex()) {
           return;