Forráskód Böngészése

HUE-4348 [indexer] Refactor format types such that all information is in one place

Aaron Peddle 9 éve
szülő
commit
a99b9da1d7

+ 39 - 0
desktop/libs/indexer/src/indexer/argument.py

@@ -0,0 +1,39 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.import logging
+class Argument():
+  _type = None
+  def __init__(self, name):
+    self._name = name
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def type(self):
+    return self._type
+
+  def to_dict(self):
+    return {"name": self._name, "type": self._type}
+
+class TextArgument(Argument):
+  _type = "text"
+
+class CheckboxArgument(Argument):
+  _type = "checkbox"
+
+class MappingArgument(Argument):
+  _type = "mapping"

+ 44 - 6
desktop/libs/indexer/src/indexer/file_format.py

@@ -19,9 +19,18 @@ import itertools
 import logging
 
 from indexer.fields import Field, guess_field_type_from_samples
+from indexer.argument import TextArgument, CheckboxArgument
 
 LOG = logging.getLogger(__name__)
 
+def get_format_types():
+  return [
+    CSVFormat,
+    HueFormat
+  ]
+
+def get_format_mapping():
+  return dict([(format_.get_name(), format_) for format_ in get_format_types()])
 
 def _valid_csv_format(format_):
   valid_field_separator = "fieldSeparator" in format_ and len(format_["fieldSeparator"]) == 1
@@ -32,15 +41,12 @@ def _valid_csv_format(format_):
   return valid_has_header and valid_quote_char and valid_record_separator and valid_field_separator
 
 def get_file_format_instance(file_stream, format_=None):
-  format_mapping = {
-    "csv": CSVFormat,
-    "hue": HueFormat
-  }
+  format_mapping = get_format_mapping()
 
   if format_ and "type" in format_:
     type_ = format_["type"]
   else:
-    type_ = "csv"
+    type_ = CSVFormat.get_name()
 
   if type_ in format_mapping:
     return format_mapping[type_](file_stream, format_)
@@ -48,6 +54,28 @@ def get_file_format_instance(file_stream, format_=None):
     return None
 
 class FileFormat(object):
+  _name = "undefined"
+  _args = []
+
+  @classmethod
+  def get_name(cls):
+    return cls._name
+
+  @classmethod
+  def get_arguments(cls):
+    return cls._args
+
+  @classmethod
+  def _valid_format(cls, format_):
+    return format_ and all([arg.name in format_ for arg in cls.get_arguments()])
+
+  @classmethod
+  def format_info(cls):
+    return {
+      "name": cls.get_name(),
+      "args": [arg.to_dict() for arg in cls.get_arguments()]
+    }
+
   def __init__(self):
     pass
 
@@ -84,6 +112,8 @@ class FileFormat(object):
     return obj
 
 class HueFormat(FileFormat):
+  _name = "hue"
+
   def __init__(self, file_stream, format_):
     self._fields = [
       Field("date", "date"),
@@ -98,12 +128,20 @@ class HueFormat(FileFormat):
     return self._fields
 
 class CSVFormat(FileFormat):
+  _name = "csv"
+  _args = [
+    TextArgument("fieldSeparator"),
+    TextArgument("recordSeparator"),
+    TextArgument("quoteChar"),
+    CheckboxArgument("hasHeader")
+  ]
+
   def __init__(self, file_stream, format_=None):
     file_stream.seek(0)
     sample = '\n'.join(file_stream.read(1024*1024*5).splitlines())
     file_stream.seek(0)
 
-    if format_:
+    if self._valid_format(format_):
       self._delimiter = format_["fieldSeparator"].encode('utf-8')
       self._line_terminator = format_["recordSeparator"].encode('utf-8')
       self._quote_char = format_["quoteChar"].encode('utf-8')

+ 25 - 40
desktop/libs/indexer/src/indexer/operations.py

@@ -13,22 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.import logging
-
-class Argument():
-  def __init__(self, type_, name):
-    self._name = name
-    self._type = type_
-
-  @property
-  def name(self):
-    return self._name
-
-  @property
-  def type(self):
-    return self._type
-
-  def to_dict(self):
-    return {"name": self._name, "type": self._type}
+from indexer.argument import TextArgument, CheckboxArgument, MappingArgument
 
 class Operator():
   def __init__(self, name, args, output_type):
@@ -55,66 +40,66 @@ OPERATORS = [
   Operator(
     name="split",
     args=[
-      Argument("text", "splitChar")
+      TextArgument("splitChar")
     ],
     output_type="custom_fields"
   ),
   Operator(
     name="grok",
     args=[
-      Argument("text", "regexp")
+      TextArgument("regexp")
     ],
     output_type="custom_fields"
   ),
   Operator(
     name="convert_date",
     args=[
-      Argument("text", "format")
+      TextArgument("format")
     ],
     output_type="inplace"
   ),
   Operator(
     name="extract_uri_components",
     args=[
-      Argument("checkbox", "authority"),
-      Argument("checkbox", "fragment"),
-      Argument("checkbox", "host"),
-      Argument("checkbox", "path"),
-      Argument("checkbox", "port"),
-      Argument("checkbox", "query"),
-      Argument("checkbox", "scheme"),
-      Argument("checkbox", "scheme_specific_path"),
-      Argument("checkbox", "user_info")
+      CheckboxArgument("authority"),
+      CheckboxArgument("fragment"),
+      CheckboxArgument("host"),
+      CheckboxArgument("path"),
+      CheckboxArgument("port"),
+      CheckboxArgument("query"),
+      CheckboxArgument("scheme"),
+      CheckboxArgument("scheme_specific_path"),
+      CheckboxArgument("user_info")
     ],
     output_type="checkbox_fields"
   ),
   Operator(
     name="geo_ip",
     args=[
-      Argument("checkbox", "/country/iso_code"),
-      Argument("checkbox", "/country/names/en"),
-      Argument("checkbox", "/subdivisions[]/names/en"),
-      Argument("checkbox", "/subdivisions[]/iso_code"),
-      Argument("checkbox", "/city/names/en"),
-      Argument("checkbox", "/postal/code"),
-      Argument("checkbox", "/location/latitude"),
-      Argument("checkbox", "/location/longitude"),
+      CheckboxArgument("/country/iso_code"),
+      CheckboxArgument("/country/names/en"),
+      CheckboxArgument("/subdivisions[]/names/en"),
+      CheckboxArgument("/subdivisions[]/iso_code"),
+      CheckboxArgument("/city/names/en"),
+      CheckboxArgument("/postal/code"),
+      CheckboxArgument("/location/latitude"),
+      CheckboxArgument("/location/longitude"),
     ],
     output_type="checkbox_fields"
   ),
   Operator(
     name="translate",
     args=[
-      Argument("text", "default"),
-      Argument("mapping", "mapping")
+      TextArgument("default"),
+      MappingArgument("mapping")
     ],
     output_type="inplace"
   ),
   Operator(
     name="find_replace",
     args=[
-      Argument("text", "find"),
-      Argument("text", "replace")
+      TextArgument("find"),
+      TextArgument("replace")
     ],
     output_type="inplace"
   ),

+ 50 - 71
desktop/libs/indexer/src/indexer/templates/indexer.mako

@@ -57,10 +57,10 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
     <div data-bind="visible: createWizard.fileFormat().show">
       <div data-bind="with: createWizard.fileFormat().format">
-          <h3>${_('File Type')}: <select data-bind="options: $root.createWizard.fileTypes, value: type"></select>
+          <h3>${_('File Type')}: <select data-bind="options: $root.createWizard.fileTypes.map(function(f){return f.name}), value: type"></select>
           </h3>
 
-          <!-- ko template: {name: 'format-settings-'+type()}--><!-- /ko -->
+          <!-- ko template: {name: 'format-settings'}--><!-- /ko -->
         </div>
 
         <h3>${_('Fields')}</h3>
@@ -107,18 +107,11 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
   </div>
 </script>
 
-<script type="text/html" id="format-settings-hue">
-</script>
-
-<script type="text/html" id="format-settings-csv">
-  <h4>${_('Has Header')}:</h4>
-  <input type="checkbox" data-bind="checked: hasHeader">
-  <h4>${_('Quote Character')}:</h4>
-  <input data-bind="value: quoteChar">
-  <h4>${_('Record Separator')}:</h4>
-  <input data-bind="value: recordSeparator">
-  <h4>${_('Field Separator')}:</h4>
-  <input data-bind="value: fieldSeparator">
+<script type="text/html" id="format-settings">
+  <!-- ko foreach: {data: getArguments(), as: 'argument'} -->
+    <h4 data-bind="text: argument.name"></h4>
+    <!-- ko template: {name: 'arg-'+argument.type, data:{name: argument.name, value: $parent[argument.name]}}--><!-- /ko -->
+  <!-- /ko -->
 </script>
 
 <script type="text/html" id="field-template">
@@ -135,7 +128,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
 <script type="text/html" id="operation-template">
   <div><select data-bind="options: $root.createWizard.operationTypes.map(function(o){return o.name});, value: operation.type"></select>
-  <!-- ko template: "operation-args-template" --><!-- /ko -->
+  <!-- ko template: "args-template" --><!-- /ko -->
     <!-- ko if: operation.settings().outputType() == "custom_fields" -->
       <input type="number" data-bind="value: operation.numExpectedFields">
     <!-- /ko -->
@@ -167,31 +160,31 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
   <!--/ko -->
 </script>
 
-<script type="text/html" id="operation-args-template">
+<script type="text/html" id="args-template">
   <!-- ko foreach: {data: operation.settings().getArguments(), as: 'argument'} -->
-    <!-- ko template: {name: 'operation-arg-'+argument.type, data:{operation: $parent.operation, argVal: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
+    <h4 data-bind="text: name"></h4>
+    <!-- ko template: {name: 'arg-'+argument.type, data:{name: argument.name, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
   <!-- /ko -->
 
 </script>
 
-<script type="text/html" id="operation-arg-text">
-  <input type="text" data-bind="attr: {placeholder: argument.name}, value: argVal">
+<script type="text/html" id="arg-text">
+  <input type="text" data-bind="attr: {placeholder: name}, value: value">
 </script>
 
-<script type="text/html" id="operation-arg-checkbox">
-  <h4 data-bind="text: argument.name"></h4>
-  <input type="checkbox" data-bind="checked: argVal">
+<script type="text/html" id="arg-checkbox">
+  <input type="checkbox" data-bind="checked: value">
 </script>
 
-<script type="text/html" id="operation-arg-mapping">
+<script type="text/html" id="arg-mapping">
   <!-- ko foreach: argVal-->
     <div>
       <input type="text" data-bind="value: key, attr: {placeholder: 'key'}">
       <input type="text" data-bind="value: value, attr: {placeholder: 'value'}">
-      <button class="btn" data-bind="click: function(){$parent.operation.settings().mapping.remove($data)}">${_('Remove Pair')}</button>
+      <button class="btn" data-bind="click: function(){$parent.value.remove($data)}">${_('Remove Pair')}</button>
     </div>
   <!-- /ko -->
-  <button class="btn" data-bind="click: operation.addPair">${_('Add Pair')}</button>
+  <button class="btn" data-bind="click: function(){value.push({key: ko.observable(''), value: ko.observable('')})}">${_('Add Pair')}</button>
   <br>
 </script>
 
@@ -308,58 +301,45 @@ var getNewFieldName = function(){
     self.type.subscribe(function(newType){
       init();
     });
-
-    self.addPair = function(){
-      self.settings().mapping.push({key: ko.observable(""), value: ko.observable("")});
-    }
   }
 
-  var getFileFormat = function(type){
-    if(type == "csv"){
-      return new CsvFileType();
-    }
-    else if(type == "hue"){
-      return new HueFileType();
-    }
-  }
-
-  var FileType = function(){
+  var FileType = function(typeName, args){
     var self = this;
+    var type;
 
-    self.loadFromObj = function(args){
-      for (var attr in args){
-        self[attr] = ko.mapping.fromJS(args[attr]);
-      }
-    }
-  }
-
-  var HueFileType = function(args){
-    var self = new FileType();
-
-    self.type = ko.observable("hue");
+    var init = function(){
+      self.type = ko.observable(typeName);
 
-    if(args) self.loadFromObj(args);
+      var types = viewModel.createWizard.fileTypes
+      for(var i = 0; i < types.length; i++){
+        if(types[i].name == typeName){
+          type = types[i];
+          break;
+        }
+      }
 
-    return self;
-  }
+      for(var i = 0; i < type.args.length; i++){
+        self[type.args[i].name] = ko.observable();
+      }
 
-  var CsvFileType = function(args){
-    var self = new FileType();
+      if(args) loadFromObj(args);
 
-    self.quoteChar = ko.observable('"');
-    self.recordSeparator = ko.observable("\\n");
-    self.type = ko.observable("csv");
-    self.hasHeader = ko.observable(false);
-    self.fieldSeparator = ko.observable(',');
+      for(var i = 0; i < type.args.length; i++){
+        self[type.args[i].name].subscribe(viewModel.createWizard.guessFieldTypes);
+      }
+    }
 
-    if(args) self.loadFromObj(args);
+    var loadFromObj = function(args){
+      for (var attr in args){
+        self[attr] = ko.mapping.fromJS(args[attr]);
+      }
+    }
 
-    self.quoteChar.subscribe(viewModel.createWizard.guessFieldTypes);
-    self.recordSeparator.subscribe(viewModel.createWizard.guessFieldTypes);
-    self.hasHeader.subscribe(viewModel.createWizard.guessFieldTypes);
-    self.fieldSeparator.subscribe(viewModel.createWizard.guessFieldTypes);
+    self.getArguments = function(){
+      return type.args;
+    }
 
-    return self;
+    init();
   }
 
   var File_Format = function (vm) {
@@ -380,8 +360,8 @@ var getNewFieldName = function(){
 
     self.operationTypes = ${operators_json | n};
 
-    self.fieldTypes = ko.observableArray(${fields_json | n});
-    self.fileTypes = ["csv","hue"];
+    self.fieldTypes = ${fields_json | n};
+    self.fileTypes = ${file_types_json | n};
 
 
     self.show = ko.observable(true);
@@ -411,7 +391,7 @@ var getNewFieldName = function(){
 
       if(self.fileFormat().format().type){
         self.fileFormat().format().type.subscribe(function(newType){
-          self.fileFormat().format(getFileFormat(newType));
+          self.fileFormat().format(new FileType(newType));
         });
       }
     });
@@ -421,8 +401,7 @@ var getNewFieldName = function(){
       $.post("${ url('indexer:guess_format') }", {
         "fileFormat": ko.mapping.toJSON(self.fileFormat)
       }, function(resp) {
-
-        self.fileFormat().format(new CsvFileType(resp));
+        self.fileFormat().format(new FileType(resp['type'], resp));
 
         self.fileFormat().show(true);
 

+ 3 - 1
desktop/libs/indexer/src/indexer/views.py

@@ -26,6 +26,7 @@ from indexer.controller2 import IndexController
 from indexer.management.commands import indexer_setup
 from indexer.fields import FIELD_TYPES
 from indexer.operations import OPERATORS
+from indexer.file_format import get_format_types
 
 LOG = logging.getLogger(__name__)
 
@@ -55,7 +56,8 @@ def indexer(request):
   return render('indexer.mako', request, {
       'indexes_json': json.dumps(indexes),
       'fields_json' : json.dumps([field.name for field in FIELD_TYPES]),
-      'operators_json' : json.dumps([operator.to_dict() for operator in OPERATORS])
+      'operators_json' : json.dumps([operator.to_dict() for operator in OPERATORS]),
+      'file_types_json' : json.dumps([format_.format_info() for format_ in get_format_types()])
   })
 
 def install_examples(request, is_redirect=False):