Sfoglia il codice sorgente

HUE-2173 [search] Harmonize indexer to use simpler field types

Romain Rigaux 8 anni fa
parent
commit
8e6b0ce

+ 1 - 1
desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.ko.js

@@ -1635,7 +1635,7 @@ var QueryResult = function (vm, initial) { // Similar to to Notebook Snippet
 
 
 var DATE_TYPES = ['date', 'tdate', 'timestamp', 'pdate'];
-var NUMBER_TYPES = ['int', 'tint', 'pint', 'long', 'tlong', 'tlong', 'float', 'tfloat', 'pfloat', 'double', 'tdouble', 'pdouble', 'currency'];
+var NUMBER_TYPES = ['int', 'tint', 'pint', 'long', 'tlong', 'plong', 'float', 'tfloat', 'pfloat', 'double', 'tdouble', 'pdouble', 'currency'];
 var FLOAT_TYPES = ['float', 'tfloat', 'pfloat', 'double', 'tdouble', 'pdouble'];
 var GEO_TYPES = ['SpatialRecursivePrefixTreeFieldType'];
 

+ 1 - 1
desktop/libs/indexer/src/indexer/api.py

@@ -162,7 +162,7 @@ def collections_create(request):
         table = request.POST.get('table')
         columns = [field['name'] for field in collection.get('fields', [])]
 
-        searcher.update_data_from_hive(db, collection.get('name'), database, table, columns)
+        searcher.update_data_from_hive(db, collection.get('name'), database, table, columns) # No up to date
 
       response['status'] = 0
       response['message'] = _('Collection created!')

+ 1 - 1
desktop/libs/indexer/src/indexer/controller.py

@@ -282,7 +282,7 @@ class CollectionManagerController(object):
         has_more = result['has_more']
 
         if result['data']:
-          kwargs = {'rowid': 'id'}
+          kwargs = {}
           dataset = tablib.Dataset()
           dataset.append(columns)
           for i, row in enumerate(result['data']):

+ 1 - 1
desktop/libs/indexer/src/indexer/fields.py

@@ -76,7 +76,7 @@ class Field(object):
     }
 
 FIELD_TYPES = [
-  FieldType('text_en', "^[\\s\\S]*$", heuristic_regex="^[\\s\\S]{101,}$"),
+  FieldType('text_general', "^[\\s\\S]*$", heuristic_regex="^[\\s\\S]{101,}$"),
   FieldType('string', "^[\\s\\S]*$", heuristic_regex="^[\\s\\S]{1,100}$"),
   FieldType('double', "^([+-]?[0-9]+(\.[0-9]+)?(E[+-]?[0-9]+)?)$"),
   FieldType('long', "^(?:[+-]?(?:[0-9]+))$"),

+ 5 - 5
desktop/libs/indexer/src/indexer/file_format.py

@@ -214,7 +214,7 @@ class HueLogFormat(GrokkedFormat):
       Field("component", "string"),
       Field("log_level", "string"),
       Field("details", "string"),
-      Field("message", "text_en"),
+      Field("message", "text_general"),
       Field("ip", "string", [geo_ip_operation]),
       Field("user", "string"),
       Field("http_method", "string"),
@@ -246,7 +246,7 @@ class ApacheCombinedFormat(GrokLineFormat):
       Field("response", "long"),
       Field("bytes", "long"),
       Field("referrer", "string"),
-      Field("field_line", "text_en")
+      Field("field_line", "text_general")
     ]
 
 
@@ -262,8 +262,8 @@ class RubyLogFormat(GrokLineFormat):
       Field("pid", "long"),
       Field("loglevel", "string"),
       Field("progname", "string"),
-      Field("message", "text_en"),
-      Field("field_line", "text_en")
+      Field("message", "text_general"),
+      Field("field_line", "text_general")
     ]
 
 
@@ -281,7 +281,7 @@ class SyslogFormat(GrokLineFormat):
       Field("logsource", "string"),
       Field("program", "string"),
       Field("pid", "string"),
-      Field("message", "text_en"),
+      Field("message", "text_general"),
     ]
 
 

+ 5 - 7
desktop/libs/indexer/src/indexer/indexers/morphline.py

@@ -127,13 +127,10 @@ class MorphlineIndexer(object):
 
   def guess_field_types(self, data):
     file_format = get_file_format_instance(data['file'], data['format'])
-    fields = file_format.get_fields() if file_format else {'columns': []}
-    for field in fields['columns']:
-      self._port_field_types(field)
-    return fields
+    return file_format.get_fields() if file_format else {'columns': []}
 
   # Breadth first ordering of fields
-  def get_field_list(self, field_data):
+  def get_field_list(self, field_data, is_converting_types=False):
     fields = []
 
     queue = deque(field_data)
@@ -141,7 +138,8 @@ class MorphlineIndexer(object):
     while len(queue):
       curr_field = queue.popleft()
       curr_field['type'] = curr_field['type']
-      self._port_field_types(curr_field)
+      if is_converting_types:
+        self._port_field_types(curr_field)
       fields.append(curr_field)
 
       for operation in curr_field["operations"]:
@@ -191,7 +189,7 @@ class MorphlineIndexer(object):
 
     properties = {
       "collection_name": collection_name,
-      "fields": self.get_field_list(data['columns']),
+      "fields": self.get_field_list(data['columns'], is_converting_types=True),
       "num_base_fields": len(data['columns']),
       "uuid_name" : uuid_name,
       "get_regex": MorphlineIndexer._get_regex_for_type,

+ 4 - 4
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -1040,12 +1040,12 @@ ${ assist.assistPanel() }
     var MAPPINGS = {
       SOLR_TO_HIVE: {
         "string": "string",
-        "long": "bigint",
-        "double": "double",
-        "date": "timestamp"
+        "plong": "bigint",
+        "pdouble": "double",
+        "pdate": "timestamp"
       },
       HIVE_TO_SOLR: {
-        "bigint": "long"
+        "bigint": "plong"
       },
       get: function(type, key, defaultValue) {
         return type[key] || defaultValue

+ 1 - 1
desktop/libs/indexer/src/indexer/tests.py

@@ -89,7 +89,7 @@ class TestIndexerWithSolr:
 
   def test_create_and_delete_collection(self):
     name = get_db_prefix(name='solr') + 'test_create_collection'
-    fields = [{'name': 'id', 'type': 'string'}, {'name': 'my_text', 'type': 'text_en'}]
+    fields = [{'name': 'id', 'type': 'string'}, {'name': 'my_text', 'type': 'text_general'}]
 
     # We get exceptions if problems in both case there
     try: