Переглянути джерело

HUE-4355 [indexer] Add i18n for arguments and extension based file type guessing

Aaron Peddle 9 роки тому
батько
коміт
a458f7708b

+ 13 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -58,7 +58,12 @@ def guess_format(request):
 
   indexer = Indexer(request.user, request.fs)
   stream = request.fs.open(file_format["path"])
-  format_ = indexer.guess_format({"file":stream})
+  format_ = indexer.guess_format({
+    "file":{
+      "stream":stream,
+      "name":file_format['path']
+      }
+    })
   _convert_format(format_)
 
   return JsonResponse(format_)
@@ -68,7 +73,13 @@ def guess_field_types(request):
   indexer = Indexer(request.user, request.fs)
   stream = request.fs.open(file_format["path"])
   _convert_format(file_format["format"], inverse = True)
-  format_ = indexer.guess_field_types({"file":stream, "format":file_format['format']})
+  format_ = indexer.guess_field_types({
+    "file":{
+      "stream":stream,
+      "name":file_format['path']
+      },
+    "format":file_format['format']
+    })
 
   return JsonResponse(format_)
 

+ 5 - 2
desktop/libs/indexer/src/indexer/argument.py

@@ -13,10 +13,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.import logging
+from django.utils.translation import ugettext as _
+
 class Argument():
   _type = None
-  def __init__(self, name):
+  def __init__(self, name, description=None):
     self._name = name
+    self._description = _(description if description else name)
 
   @property
   def name(self):
@@ -27,7 +30,7 @@ class Argument():
     return self._type
 
   def to_dict(self):
-    return {"name": self._name, "type": self._type}
+    return {"name": self._name, "type": self._type, "description": self._description}
 
 class TextArgument(Argument):
   _type = "text"

+ 65 - 32
desktop/libs/indexer/src/indexer/file_format.py

@@ -18,6 +18,8 @@ import operator
 import itertools
 import logging
 
+from django.utils.translation import ugettext as _
+
 from indexer.fields import Field, guess_field_type_from_samples
 from indexer.argument import TextArgument, CheckboxArgument
 
@@ -32,57 +34,67 @@ def get_format_types():
 def get_format_mapping():
   return dict([(format_.get_name(), format_) for format_ in get_format_types()])
 
-def _valid_csv_format(format_):
-  valid_field_separator = "fieldSeparator" in format_ and len(format_["fieldSeparator"]) == 1
-  valid_record_separator = "recordSeparator" in format_ and len(format_["recordSeparator"]) == 1
-  valid_quote_char = "quoteChar" in format_ and len(format_["quoteChar"]) == 1
-  valid_has_header = "recordSeparator" in format_
-
-  return valid_has_header and valid_quote_char and valid_record_separator and valid_field_separator
+def get_file_format_instance(file, format_=None):
+  file_stream = file['stream']
+  file_extension = file['name'].split('.')[-1] if '.' in file['name'] else ''
 
-def get_file_format_instance(file_stream, format_=None):
   format_mapping = get_format_mapping()
 
   if format_ and "type" in format_:
     type_ = format_["type"]
-  else:
-    type_ = CSVFormat.get_name()
+    if type_ in format_mapping:
+      if format_mapping[type_].valid_format(format_):
+        return format_mapping[type_](file_stream, format_)
+      else:
+        return None
+
+  matches = [type_ for type_ in get_format_types() if file_extension in type_.get_extensions()]
 
-  if type_ in format_mapping:
-    return format_mapping[type_](file_stream, format_)
-  else:
-    return None
+  return (matches[0] if matches else get_format_types()[0])(file_stream, format_)
 
 class FileFormat(object):
-  _name = "undefined"
+  _name = None
+  _description = None
+  _customizable = True
   _args = []
+  _extensions = []
+
+  @classmethod
+  def get_extensions(cls):
+    return cls._extensions
 
   @classmethod
   def get_name(cls):
     return cls._name
 
+  @classmethod
+  def get_description(cls):
+    return cls._description
+
   @classmethod
   def get_arguments(cls):
     return cls._args
 
   @classmethod
-  def _valid_format(cls, format_):
+  def is_customizable(cls):
+    return cls._customizable
+
+  @classmethod
+  def valid_format(cls, format_):
     return format_ and all([arg.name in format_ for arg in cls.get_arguments()])
 
   @classmethod
   def format_info(cls):
     return {
       "name": cls.get_name(),
-      "args": [arg.to_dict() for arg in cls.get_arguments()]
+      "args": [arg.to_dict() for arg in cls.get_arguments()],
+      "description": cls.get_description(),
+      "isCustomizable": cls.is_customizable()
     }
 
   def __init__(self):
     pass
 
-  @property
-  def format_(self):
-    pass
-
   @property
   def sample(self):
     pass
@@ -92,7 +104,7 @@ class FileFormat(object):
     return []
 
   def get_format(self):
-    return self.format_
+    return {"type": self.get_name()}
 
   def get_fields(self):
     obj = {}
@@ -105,7 +117,7 @@ class FileFormat(object):
   def to_dict(self):
     obj = {}
 
-    obj['format'] = self.format_
+    obj['format'] = self.get_format()
     obj['columns'] = [field.to_dict() for field in self.fields]
     obj['sample'] = self.sample
 
@@ -113,6 +125,9 @@ class FileFormat(object):
 
 class HueFormat(FileFormat):
   _name = "hue"
+  _description = _("Hue Log File")
+  _customizable = False
+  _extensions = ["log"]
 
   def __init__(self, file_stream, format_):
     self._fields = [
@@ -129,19 +144,35 @@ class HueFormat(FileFormat):
 
 class CSVFormat(FileFormat):
   _name = "csv"
+  _description = _("CSV File")
   _args = [
-    TextArgument("fieldSeparator"),
-    TextArgument("recordSeparator"),
-    TextArgument("quoteChar"),
-    CheckboxArgument("hasHeader")
+    TextArgument("fieldSeparator", "Field Separator"),
+    TextArgument("recordSeparator", "Record Separator"),
+    TextArgument("quoteChar", "Quote Character"),
+    CheckboxArgument("hasHeader", "Has Header")
   ]
+  _extensions = ["csv", "tsv"]
+
+  @classmethod
+  def _valid_character(self, char):
+    return isinstance(char, basestring) and len(char) == 1
+
+  @classmethod
+  def valid_format(cls, format_):
+    valid = super(CSVFormat, cls).valid_format(format_)
+    valid = valid and cls._valid_character(format_["fieldSeparator"])
+    valid = valid and cls._valid_character(format_["recordSeparator"])
+    valid = valid and cls._valid_character(format_["quoteChar"])
+    valid = valid and isinstance(format_["hasHeader"], bool)
+
+    return valid
 
   def __init__(self, file_stream, format_=None):
     file_stream.seek(0)
     sample = '\n'.join(file_stream.read(1024*1024*5).splitlines())
     file_stream.seek(0)
 
-    if self._valid_format(format_):
+    if self.valid_format(format_):
       self._delimiter = format_["fieldSeparator"].encode('utf-8')
       self._line_terminator = format_["recordSeparator"].encode('utf-8')
       self._quote_char = format_["quoteChar"].encode('utf-8')
@@ -190,15 +221,17 @@ class CSVFormat(FileFormat):
   def quote_char(self):
     return self._quote_char
 
-  @property
-  def format_(self):
-    return {
-      "type":"csv",
+  def get_format(self):
+    format_ = super(CSVFormat, self).get_format()
+    specific_format = {
       "fieldSeparator":self.delimiter,
       "recordSeparator":self.line_terminator,
       "quoteChar":self.quote_char,
       "hasHeader":self._has_header
     }
+    format_.update(specific_format)
+
+    return format_
 
   def _guess_dialect(self, sample):
     sniffer = csv.Sniffer()

+ 25 - 24
desktop/libs/indexer/src/indexer/operations.py

@@ -13,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.import logging
+from django.utils.translation import ugettext as _
 from indexer.argument import TextArgument, CheckboxArgument, MappingArgument
 
 class Operator():
@@ -40,66 +41,66 @@ OPERATORS = [
   Operator(
     name="split",
     args=[
-      TextArgument("splitChar")
+      TextArgument("splitChar", _("Split Chararacter"))
     ],
     output_type="custom_fields"
   ),
   Operator(
     name="grok",
     args=[
-      TextArgument("regexp")
+      TextArgument("regexp", _("Regular Expression"))
     ],
     output_type="custom_fields"
   ),
   Operator(
     name="convert_date",
     args=[
-      TextArgument("format")
+      TextArgument("format", _("Date Format (eg: yyyy/MM/dd)"))
     ],
     output_type="inplace"
   ),
   Operator(
     name="extract_uri_components",
     args=[
-      CheckboxArgument("authority"),
-      CheckboxArgument("fragment"),
-      CheckboxArgument("host"),
-      CheckboxArgument("path"),
-      CheckboxArgument("port"),
-      CheckboxArgument("query"),
-      CheckboxArgument("scheme"),
-      CheckboxArgument("scheme_specific_path"),
-      CheckboxArgument("user_info")
+      CheckboxArgument("authority", _("Authority")),
+      CheckboxArgument("fragment", _("Fragment")),
+      CheckboxArgument("host", _("Host")),
+      CheckboxArgument("path", _("Path")),
+      CheckboxArgument("port", _("Port")),
+      CheckboxArgument("query", _("Query")),
+      CheckboxArgument("scheme", _("Scheme")),
+      CheckboxArgument("scheme_specific_path", _("Scheme Specific Path")),
+      CheckboxArgument("user_info", _("User Info"))
     ],
     output_type="checkbox_fields"
   ),
   Operator(
     name="geo_ip",
     args=[
-      CheckboxArgument("/country/iso_code"),
-      CheckboxArgument("/country/names/en"),
-      CheckboxArgument("/subdivisions[]/names/en"),
-      CheckboxArgument("/subdivisions[]/iso_code"),
-      CheckboxArgument("/city/names/en"),
-      CheckboxArgument("/postal/code"),
-      CheckboxArgument("/location/latitude"),
-      CheckboxArgument("/location/longitude"),
+      CheckboxArgument("/country/iso_code", _("ISO Code")),
+      CheckboxArgument("/country/names/en", _("Country Name")),
+      CheckboxArgument("/subdivisions[]/names/en", _("Subdivisions Names")),
+      CheckboxArgument("/subdivisions[]/iso_code", _("Subdivisons ISO Code")),
+      CheckboxArgument("/city/names/en", _("City Name")),
+      CheckboxArgument("/postal/code", _("Postal Code")),
+      CheckboxArgument("/location/latitude", _("Latitude")),
+      CheckboxArgument("/location/longitude", _("Longitude")),
     ],
     output_type="checkbox_fields"
   ),
   Operator(
     name="translate",
     args=[
-      TextArgument("default"),
-      MappingArgument("mapping")
+      TextArgument("default", "Default Value (if no match found)"),
+      MappingArgument("mapping", _("Mapping"))
     ],
     output_type="inplace"
   ),
   Operator(
     name="find_replace",
     args=[
-      TextArgument("find"),
-      TextArgument("replace")
+      TextArgument("find", _("Find")),
+      TextArgument("replace", _("Replace"))
     ],
     output_type="inplace"
   ),

+ 55 - 36
desktop/libs/indexer/src/indexer/templates/indexer.mako

@@ -56,38 +56,40 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
 
     <div data-bind="visible: createWizard.fileFormat().show">
+      <h3>${_('File Type')}: <select data-bind="options: $root.createWizard.fileTypes, optionsText: 'description', value: $root.createWizard.fileType"></select>
+      </h3>
       <div data-bind="with: createWizard.fileFormat().format">
-          <h3>${_('File Type')}: <select data-bind="options: $root.createWizard.fileTypes.map(function(f){return f.name}), value: type"></select>
-          </h3>
-
-          <!-- ko template: {name: 'format-settings'}--><!-- /ko -->
-        </div>
-
-        <h3>${_('Fields')}</h3>
-        <div data-bind="foreach: createWizard.fileFormat().columns">
-          <div data-bind="template: { name:'field-template',data:$data}"></div>
-        </div>
-
-        <h3>${_('Preview')}</h3>
-        <table style="margin:auto;text-align:left">
-          <thead>
-            <tr data-bind="foreach: createWizard.fileFormat().columns">
-              <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
-            </tr>
-          </thead>
-          <tbody data-bind="foreach: createWizard.sample">
-            <tr data-bind="foreach: $data">
-              <!-- ko if: $index() < $root.createWizard.fileFormat().columns().length -->
-                <td data-bind="visible: $root.createWizard.fileFormat().columns()[$index()].keep, text: $data">
-                </td>
-
-                <!-- ko with: $root.createWizard.fileFormat().columns()[$index()] -->
-                  <!-- ko template: 'output-generated-field-data-template' --> <!-- /ko -->
+
+        <!-- ko template: {name: 'format-settings'}--><!-- /ko -->
+      </div>
+
+        <!-- ko if: createWizard.fileFormat().format() && createWizard.fileFormat().format().isCustomizable() -->
+          <h3>${_('Fields')}</h3>
+          <div data-bind="foreach: createWizard.fileFormat().columns">
+            <div data-bind="template: { name:'field-template',data:$data}"></div>
+          </div>
+
+          <h3>${_('Preview')}</h3>
+          <table style="margin:auto;text-align:left">
+            <thead>
+              <tr data-bind="foreach: createWizard.fileFormat().columns">
+                <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
+              </tr>
+            </thead>
+            <tbody data-bind="foreach: createWizard.sample">
+              <tr data-bind="foreach: $data">
+                <!-- ko if: $index() < $root.createWizard.fileFormat().columns().length -->
+                  <td data-bind="visible: $root.createWizard.fileFormat().columns()[$index()].keep, text: $data">
+                  </td>
+
+                  <!-- ko with: $root.createWizard.fileFormat().columns()[$index()] -->
+                    <!-- ko template: 'output-generated-field-data-template' --> <!-- /ko -->
+                  <!-- /ko -->
                 <!-- /ko -->
-              <!-- /ko -->
-            </tr>
-          </tbody>
-        </table>
+              </tr>
+            </tbody>
+          </table>
+        <!-- /ko -->
 
         <br><hr><br>
 
@@ -109,8 +111,8 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
 <script type="text/html" id="format-settings">
   <!-- ko foreach: {data: getArguments(), as: 'argument'} -->
-    <h4 data-bind="text: argument.name"></h4>
-    <!-- ko template: {name: 'arg-'+argument.type, data:{name: argument.name, value: $parent[argument.name]}}--><!-- /ko -->
+    <h4 data-bind="text: argument.description"></h4>
+    <!-- ko template: {name: 'arg-'+argument.type, data:{description: argument.description, value: $parent[argument.name]}}--><!-- /ko -->
   <!-- /ko -->
 </script>
 
@@ -162,14 +164,14 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
 <script type="text/html" id="args-template">
   <!-- ko foreach: {data: operation.settings().getArguments(), as: 'argument'} -->
-    <h4 data-bind="text: name"></h4>
-    <!-- ko template: {name: 'arg-'+argument.type, data:{name: argument.name, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
+    <h4 data-bind="text: description"></h4>
+    <!-- ko template: {name: 'arg-'+argument.type, data:{description: argument.description, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
   <!-- /ko -->
 
 </script>
 
 <script type="text/html" id="arg-text">
-  <input type="text" data-bind="attr: {placeholder: name}, value: value">
+  <input type="text" data-bind="attr: {placeholder: description}, value: value">
 </script>
 
 <script type="text/html" id="arg-checkbox">
@@ -311,6 +313,7 @@ var getNewFieldName = function(){
       self.type = ko.observable(typeName);
 
       var types = viewModel.createWizard.fileTypes
+
       for(var i = 0; i < types.length; i++){
         if(types[i].name == typeName){
           type = types[i];
@@ -339,6 +342,10 @@ var getNewFieldName = function(){
       return type.args;
     }
 
+    self.isCustomizable = function(){
+      return type.isCustomizable;
+    }
+
     init();
   }
 
@@ -358,6 +365,11 @@ var getNewFieldName = function(){
     var self = this;
     var guessFieldTypesXhr;
 
+    self.fileType = ko.observable();
+    self.fileType.subscribe(function(newType){
+      if(self.fileFormat().format()) self.fileFormat().format().type(newType.name);
+    });
+
     self.operationTypes = ${operators_json | n};
 
     self.fieldTypes = ${fields_json | n};
@@ -388,6 +400,12 @@ var getNewFieldName = function(){
 
     self.fileFormat().format.subscribe(function(){
       self.guessFieldTypes();
+      for(var i = 0; i < self.fileTypes.length; i++){
+        if(self.fileTypes[i].name == self.fileFormat().format().type()){
+          self.fileType(self.fileTypes[i]);
+          break;
+        }
+      }
 
       if(self.fileFormat().format().type){
         self.fileFormat().format().type.subscribe(function(newType){
@@ -401,7 +419,8 @@ var getNewFieldName = function(){
       $.post("${ url('indexer:guess_format') }", {
         "fileFormat": ko.mapping.toJSON(self.fileFormat)
       }, function(resp) {
-        self.fileFormat().format(new FileType(resp['type'], resp));
+        var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
+        self.fileFormat().format(newFormat);
 
         self.fileFormat().show(true);
 

+ 10 - 10
desktop/libs/indexer/src/indexer/tests_indexer.py

@@ -42,9 +42,9 @@ class IndexerTest():
     stream = StringIO.StringIO(IndexerTest.simpleCSVString)
     indexer = Indexer("test", None)
 
-    guessed_format = indexer.guess_format({'file': stream})
+    guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
 
-    fields = indexer.guess_field_types({"file":stream, "format": guessed_format})['columns']
+    fields = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
     # test format
     assert_equal('csv', guessed_format['type'])
     assert_equal(',', guessed_format['fieldSeparator'])
@@ -82,27 +82,27 @@ class IndexerTest():
     indexer = Indexer("test", None)
     stream = StringIO.StringIO(IndexerTest.simpleCSVString)
 
-    guessed_format = indexer.guess_format({'file': stream})
+    guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
 
     guessed_format["fieldSeparator"] = "invalid separator"
 
-    fields = indexer.guess_field_types({"file":stream, "format": guessed_format})['columns']
+    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
     assert_equal(fields, [])
 
     stream.seek(0)
-    guessed_format = indexer.guess_format({'file': stream})
+    guessed_format = indexer.guess_format({'file':  {"stream": stream, "name": "test.csv"}})
 
     guessed_format["recordSeparator"] = "invalid separator"
 
-    fields = indexer.guess_field_types({"file":stream, "format": guessed_format})['columns']
+    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
     assert_equal(fields, [])
 
     stream.seek(0)
-    guessed_format = indexer.guess_format({'file': stream})
+    guessed_format = indexer.guess_format({'file':  {"stream": stream, "name": "test.csv"}})
 
     guessed_format["quoteChar"] = "invalid quoteChar"
 
-    fields = indexer.guess_field_types({"file":stream, "format": guessed_format})['columns']
+    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
     assert_equal(fields, [])
 
   def test_end_to_end(self):
@@ -118,9 +118,9 @@ class IndexerTest():
     stream = fs.open(input_loc)
 
     # guess the format of the file
-    file_type_format = indexer.guess_format({'file': stream})
+    file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
 
-    field_types = indexer.guess_field_types({"file":stream, "format": file_type_format})
+    field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format})
 
     format_ = field_types.copy()
     format_['format'] = file_type_format