Browse Source

HUE-4474 [indexer] Let user set an existing field to unique id

Aaron Peddle 9 years ago
parent
commit
30806710b0

+ 7 - 6
desktop/libs/indexer/src/data/oozie_workspace/morphline_template.conf

@@ -44,13 +44,14 @@ morphlines : [
 
       <%include file="${'parse_%s.conf' % format_class.get_parse_type()}", args="format_settings=format_settings, format_class=format_class"/>
 
-
-      {
-        generateUUID {
-          field : ${uuid_name}
-          type : "nonSecure"
+      % if uuid_name:
+        {
+          generateUUID {
+            field : ${uuid_name}
+            type : "nonSecure"
+          }
         }
-      }
+      % endif
 
       # process operations on fields
       % for field in fields:

+ 6 - 3
desktop/libs/indexer/src/indexer/api3.py

@@ -88,9 +88,12 @@ def index_file(request):
   _convert_format(file_format["format"], inverse = True)
   collection_name = file_format["name"]
   indexer = Indexer(request.user, request.fs)
-  unique_field = indexer.get_uuid_name(file_format)
-  schema_fields = [{"name": unique_field, "type": "string"}] + \
-    indexer.get_kept_field_list(file_format['columns'])
+  unique_field = indexer.get_unique_field(file_format)
+  is_unique_generated = indexer.is_unique_generated(file_format)
+
+  schema_fields = indexer.get_kept_field_list(file_format['columns'])
+  if is_unique_generated:
+    schema_fields += [{"name": unique_field, "type": "string"}]
 
   morphline = indexer.generate_morphline_config(collection_name, file_format, unique_field)
 

+ 3 - 1
desktop/libs/indexer/src/indexer/fields.py

@@ -39,16 +39,18 @@ class FieldType():
     return pattern.match(field)
 
 class Field(object):
-  def __init__(self, name, field_type_name, operations=None):
+  def __init__(self, name="new_field", field_type_name="string", operations=None):
     self.name = name
     self.field_type_name = field_type_name
     self.keep = True
     self.operations = operations if operations else []
     self.required = False
+    self.unique = False
 
   def to_dict(self):
     return {'name': self.name,
     'type': self.field_type_name,
+    'unique': self.unique,
     'keep': self.keep,
     'operations': self.operations,
     'required': self.required}

+ 17 - 7
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -133,15 +133,25 @@ class Indexer(object):
   def get_kept_field_list(self, field_data):
     return [field for field in self.get_field_list(field_data) if field['keep']]
 
-  def get_uuid_name(self, format_):
-    base_name = "_uuid"
+  def get_unique_field(self, format_):
+    # check for a unique field
+    unique_fields = [column['name'] for column in format_['columns'] if column['unique']]
 
-    field_names = set([column['name'] for column in format_['columns']])
+    if unique_fields:
+      return unique_fields[0]
+    else:
+      base_name = "_uuid"
+      field_names = set([column['name'] for column in format_['columns']])
 
-    while base_name in field_names:
-      base_name = '_' + base_name
+      while base_name in field_names:
+        base_name = '_' + base_name
 
-    return base_name
+      return base_name
+
+  def is_unique_generated(self, format_):
+    unique_fields = [column['name'] for column in format_['columns'] if column['unique']]
+
+    return len(unique_fields) == 0
 
   @staticmethod
   def _get_regex_for_type(type_name):
@@ -149,7 +159,7 @@ class Indexer(object):
 
     return field_type.regex.replace('\\', '\\\\')
 
-  def generate_morphline_config(self, collection_name, data, uuid_name="__uuid"):
+  def generate_morphline_config(self, collection_name, data, uuid_name=None):
     """
     Input:
     data: {

+ 6 - 7
desktop/libs/indexer/src/indexer/templates/indexer.mako

@@ -202,6 +202,7 @@ ${ assist.assistPanel() }
 
 <script type="text/html" id="field-template">
   <div>
+    <span>${_('Unique')}</span><input type="checkbox" data-bind="checked: unique">
     <span>${_('Keep')}</span><input type="checkbox" data-bind="checked: keep">
     <span>${_('Required')}</span><input type="checkbox" data-bind="checked: required">
     <input type="text" data-bind="value: name"></input> - <select data-bind="options: $root.createWizard.fieldTypes, value: type"></select>
@@ -309,13 +310,11 @@ ${ assist.assistPanel() }
   }
 
   var createDefaultField = function () {
-    return {
-      name: ko.observable(getNewFieldName()),
-      type: ko.observable("string"),
-      keep: ko.observable(true),
-      required: ko.observable(true),
-      operations: ko.observableArray([])
-    }
+    var defaultField = ${default_field_type | n};
+
+    defaultField.name = getNewFieldName();
+
+    return defaultField;
   };
 
   var Operation = function (type) {

+ 6 - 2
desktop/libs/indexer/src/indexer/tests_indexer.py

@@ -254,12 +254,16 @@ class IndexerTest():
     format_['format'] = file_type_format
 
     # find a field name available to use for the record's uuid
-    unique_field = indexer.get_uuid_name(format_)
+    unique_field = indexer.get_unique_field(format_)
+    is_unique_generated = indexer.is_unique_generated(format_)
 
     # generate morphline
     morphline = indexer.generate_morphline_config(collection_name, format_, unique_field)
 
-    schema_fields = [{"name": unique_field, "type": "string"}] + indexer.get_kept_field_list(format_['columns'])
+    schema_fields = indexer.get_kept_field_list(format_['columns'])
+    if is_unique_generated:
+      schema_fields += [{"name": unique_field, "type": "string"}]
+
 
     # create the collection from the specified fields
     collection_manager = CollectionManagerController("test")

+ 3 - 2
desktop/libs/indexer/src/indexer/views.py

@@ -24,7 +24,7 @@ from desktop.lib.django_util import JsonResponse, render
 
 from indexer.controller2 import IndexController
 from indexer.management.commands import indexer_setup
-from indexer.fields import FIELD_TYPES
+from indexer.fields import FIELD_TYPES, Field
 from indexer.operations import OPERATORS
 from indexer.file_format import get_format_types
 
@@ -57,7 +57,8 @@ def indexer(request):
       'indexes_json': json.dumps(indexes),
       'fields_json' : json.dumps([field.name for field in FIELD_TYPES]),
       'operators_json' : json.dumps([operator.to_dict() for operator in OPERATORS]),
-      'file_types_json' : json.dumps([format_.format_info() for format_ in get_format_types()])
+      'file_types_json' : json.dumps([format_.format_info() for format_ in get_format_types()]),
+      'default_field_type' : json.dumps(Field().to_dict())
   })
 
 def install_examples(request, is_redirect=False):