Selaa lähdekoodia

HUE-8574 [importer] Allow audit logs to be sent to Solr

Romain Rigaux 7 vuotta sitten
vanhempi
commit
3e1463d1a4

+ 1 - 5
desktop/core/src/desktop/templates/ko_components/ko_multi_cluster_sidebar.mako

@@ -140,10 +140,6 @@ from desktop.views import _ko
           label: '${ _('Data Science') }',
           url: '/',
           icon: 'altus-icon altus-ds'
-        }, {
-          label: '${ _('Customer 360') }',
-          url: '/',
-          icon: 'fa fa-vcard-o'
         }
       ]
       }, {
@@ -153,7 +149,7 @@ from desktop.views import _ko
             url: '/',
             icon: 'altus-icon altus-environment'
           },{
-            label: '${ _('IAM') }',
+            label: '${ _('Directory') }',
             url: 'https://console.altus.cloudera.com/iam/index.html',
             icon: 'altus-icon altus-iam',
             items: [{

+ 15 - 10
desktop/libs/indexer/src/indexer/api3.py

@@ -263,6 +263,8 @@ def guess_field_types(request):
         kafkaFieldTypes = [
           'string'
         ] * len(kafkaFieldNames)
+        kafkaFieldNames.append('timeDate')
+        kafkaFieldTypes.append('date')
       else:
         kafkaFieldNames = file_format.get('kafkaFieldNames', '').split(',')
         kafkaFieldTypes = file_format.get('kafkaFieldTypes', '').split(',')
@@ -477,21 +479,23 @@ def _large_indexing(request, file_format, collection_name, query=None, start_tim
 
   client = SolrClient(user=request.user)
 
-  if not client.exists(collection_name):
+  if not client.exists(collection_name): # if destination['isTargetExisting']:
     client.create_index(
       name=collection_name,
       fields=request.POST.get('fields', schema_fields),
       unique_key_field=unique_field
       # No df currently
     )
+  else:
+    # TODO: check if format matches
+    pass
 
   if file_format['inputFormat'] == 'table':
     db = dbms.get(request.user)
     table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
     input_path = table_metadata.path_location
   elif file_format['inputFormat'] == 'stream':
-    pass
-    #job_handle = _envelope_job(request, source, destination, start_time=start_time, lib_path=destination['indexerJobLibPath'])
+    return _envelope_job(request, file_format, collection_name, start_time=start_time, lib_path=lib_path)
   elif file_format['inputFormat'] == 'file':
     input_path = '${nameNode}%s' % urllib.unquote(file_format["path"])
   else:
@@ -571,13 +575,14 @@ def _envelope_job(request, file_format, destination, start_time=None, lib_path=N
     elif destination['outputFormat'] == 'index':
       properties['collectionName'] = collection_name
       properties['connection'] = SOLR_URL.get()
-      if destination['isTargetExisting']:
-        # Todo: check if format matches
-        pass
-      else:
-        client = SolrClient(request.user)
-        kwargs = {}
-        _create_solr_collection(request.user, request.fs, client, destination, collection_name, kwargs)
+# No needed anymore
+#       if destination['isTargetExisting']:
+#         # Todo: check if format matches
+#         pass
+#       else:
+#         client = SolrClient(request.user)
+#         kwargs = {}
+#         _create_solr_collection(request.user, request.fs, client, destination, collection_name, kwargs)
 
   if destination['outputFormat'] == 'stream':
     manager = ManagerApi()

+ 13 - 9
desktop/libs/indexer/src/indexer/indexers/envelope.py

@@ -180,15 +180,19 @@ SPARK_KAFKA_VERSION=0.10 spark2-submit envelope.jar envelope.conf"""
               table.name = "%(output_table)s"
           }""" % properties
     elif properties['ouputFormat'] == 'index':
-      output = """
-        planner {
-            type = upstert
-        }
-        output {
-            type = solr
-            connection = "%(connection)s"
-            collection.name = "%(collectionName)s"
-        }""" % properties
+      if properties['inputFormat'] == 'stream':
+        if properties['topics'] == 'NavigatorAuditEvents':
+          output = ''
+      else:
+        output = """
+          planner {
+              type = upstert
+          }
+          output {
+              type = solr
+              connection = "%(connection)s"
+              collection.name = "%(collectionName)s"
+          }""" % properties
     elif properties['ouputFormat'] == 'stream':
       output = """
         planner {