Переглянути джерело

HUE-7409 [search] More subtle way of converting old types to new types

Romain Rigaux 8 роки тому
батько
коміт
aa71cec

+ 10 - 2
desktop/libs/indexer/src/indexer/indexers/morphline.py

@@ -126,7 +126,10 @@ class MorphlineIndexer(object):
 
   def guess_field_types(self, data):
     file_format = get_file_format_instance(data['file'], data['format'])
-    return file_format.get_fields() if file_format else {'columns': []}
+    fields = file_format.get_fields() if file_format else {'columns': []}
+    for field in fields['columns']:
+      self._port_field_types(field)
+    return fields
 
   # Breadth first ordering of fields
   def get_field_list(self, field_data):
@@ -136,7 +139,8 @@ class MorphlineIndexer(object):
 
     while len(queue):
       curr_field = queue.popleft()
-      curr_field['type'] = curr_field['type'].replace('long', 'plong').replace('double', 'pdouble').replace('date', 'pdate')
+      curr_field['type'] = curr_field['type']
+      self._port_field_types(curr_field)
       fields.append(curr_field)
 
       for operation in curr_field["operations"]:
@@ -145,6 +149,10 @@ class MorphlineIndexer(object):
 
     return fields
 
+  def _port_field_types(self, field):
+    if not field['type'].startswith('p'): # Check for automatically converting to new default Solr types
+      field['type'] = field['type'].replace('long', 'plong').replace('double', 'pdouble').replace('date', 'pdate')
+
   def get_kept_field_list(self, field_data):
     return [field for field in self.get_field_list(field_data) if field['keep']]
 

+ 1 - 1
desktop/libs/indexer/src/indexer/indexers/morphline_tests.py

@@ -109,7 +109,7 @@ class TestIndexer():
 
     guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
 
-    fields = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
+    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
     # test format
     expected_format = self.simpleCSVFormat
 

+ 1 - 0
docs/release-notes/release-notes-4.1.0.txt

@@ -24,6 +24,7 @@ Quality release
 * SQL improvements
 
 Stability
+
 * Risk alert fix suggestions
 
 Security