Browse Source

HUE-7713 [search] Support handling indexing of documents with bad fields

This is only for Solr 6+ and creation created via Hue or having a 'tolerant' updateprocessor chain available.
It will not fail the ingest until a maximum of 100 records fails to be indexed. The reason of failure (e.g.
why one of more fields are not compatible with the type) for the records will be displayed.
Romain Rigaux 8 years ago
parent
commit
7ce7954

+ 6 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -213,6 +213,7 @@ def _create_index(user, fs, client, source, destination, index_name):
   unique_key_field = destination['indexerPrimaryKey'] and destination['indexerPrimaryKey'][0] or None
   df = destination['indexerDefaultField'] and destination['indexerDefaultField'][0] or None
   kwargs = {}
+  errors = []
 
   if source['inputFormat'] not in ('manual', 'table'):
     stats = fs.stats(source["path"])
@@ -252,7 +253,10 @@ def _create_index(user, fs, client, source, destination, index_name):
   if source['inputFormat'] not in ('manual', 'table'):
     data = fs.read(source["path"], 0, MAX_UPLOAD_SIZE)
     try:
-      client.index(name=index_name, data=data, **kwargs)
+      if client.is_solr_six_or_more():
+        kwargs['processor'] = 'tolerant'
+      response = client.index(name=index_name, data=data, **kwargs)
+      errors = [error.get('message', '') for error in response['responseHeader'].get('errors', [])]
     except Exception, e:
       try:
         client.delete_index(index_name, keep_config=False)
@@ -260,7 +264,7 @@ def _create_index(user, fs, client, source, destination, index_name):
         LOG.warn('Error while cleaning-up config of failed collection creation %s: %s' % (index_name, e2))
       raise e
 
-  return {'status': 0, 'on_success_url': reverse('indexer:indexes', kwargs={'index': index_name}), 'pub_sub_url': 'assist.collections.refresh'}
+  return {'status': 0, 'on_success_url': reverse('indexer:indexes', kwargs={'index': index_name}), 'pub_sub_url': 'assist.collections.refresh', 'errors': errors}
 
 
 def _create_database(request, source, destination, start_time):

+ 9 - 0
desktop/libs/indexer/src/indexer/solr_client.py

@@ -133,6 +133,15 @@ class SolrClient(object):
               "defaults": {"df": df},
             }
           })
+
+        if self.is_solr_six_or_more():
+          self.api.update_config(name, {
+            'add-updateprocessor': {
+              "name" : "tolerant",
+              "class": "solr.TolerantUpdateProcessorFactory",
+              "maxErrors": "100"
+            }
+          })
       else:
         self._create_cloud_config(name, fields, unique_key_field, df)
         self.api.create_collection2(name, config_name=config_name, shards=shards, replication=replication)

+ 3 - 0
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -2033,6 +2033,9 @@ ${ assist.assistPanel() }
                 huePubSub.publish(resp.pub_sub_url);
               }
               $.jHueNotify.info("${ _('Creation success') }");
+              if (resp.errors) {
+                $.jHueNotify.warn("${ _('Skipped records: ') }" + resp.errors.join(', '));
+              }
               huePubSub.publish('open.link', resp.on_success_url);
             }
           } else {