Эх сурвалжийг харах

HUE-4332 [indexer] Add hue logs as a supported file type

whitespace cleanup

fixes from review
Aaron Peddle 9 жил өмнө
parent
commit
65fef05

+ 29 - 30
desktop/libs/indexer/src/data/oozie_workspace/clean_to_match_schema.conf

@@ -15,6 +15,32 @@
 # limitations under the License.from nose.tools import assert_equal
 <%page args="fields, get_regex"/>
 
+# treat empty strings as if the field wasn't in the tuple unless the type is string
+% for field in fields:
+  % if field["type"] != "string":
+    {
+      if {
+        conditions : [
+          {
+            grok {
+              expressions : {
+                "${field['name']}" : "^$"
+              }
+              extract : false
+            }
+          }
+        ]
+        then : [
+          {
+            removeFields {
+              blacklist : ["literal:${field['name']}"]
+            }
+          }
+        ]
+      }
+    }
+  % endif
+%endfor
 
 # require that all kept fields are present
 % for field in fields:
@@ -25,7 +51,7 @@
           { equals { "${field['name']}" : [] } }
         ]
         then : [
-          { logError { format : "Ignoring record because it has no ${field['name']}: {}", args : ["@{}"] } }
+          { logTrace { format : "Ignoring record because it has no ${field['name']}: {}", args : ["@{}"] } }
           { dropRecord {} }
         ]
       }
@@ -73,13 +99,13 @@
                 ]
                 then : []
                 else : [
-                  { logError { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
+                  { logTrace { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
                   { dropRecord {} }
                 ]
               }
             }
           %else:
-            { logError { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
+            { logTrace { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
             { dropRecord {} }
           %endif
         ]
@@ -88,33 +114,6 @@
   %endif
 %endfor
 
-# treat empty strings as if the field wasn't in the tuple unless the type is string
-% for field in fields:
-  % if field["type"] != "string":
-    {
-      if {
-        conditions : [
-          {
-            grok {
-              expressions : {
-                "${field['name']}" : "^$"
-              }
-              extract : false
-            }
-          }
-        ]
-        then : [
-          {
-            removeFields {
-              blacklist : ["literal:${field['name']}"]
-            }
-          }
-        ]
-      }
-    }
-  % endif
-%endfor
-
 # remove excess fields
 {
   sanitizeUnknownSolrFields {

+ 45 - 0
desktop/libs/indexer/src/data/oozie_workspace/parse_hue.conf

@@ -0,0 +1,45 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from nose.tools import assert_equal
+<%page args="format, format_character"/>
+
+{
+  readMultiLine{
+    regex: "^(?!\\[[0-9]{1,2}\\/\\w+\\/[0-9]{4}.*\\].*).*$"
+    what: previous
+    charset: UTF-8
+  }
+}
+{ logTrace { format : "Output from multiline {}", args : ["@{}"] } }
+
+{
+  grok {
+    dictionaryFiles : ["grok_dictionaries"]
+
+    expressions : {
+      message: """\[%{GREEDYDATA:date}\] %{WORD:component} +%{LOGLEVEL:log_level} %{GREEDYDATA:details}[\s\S]*"""
+    }
+  }
+}
+{ logTrace { format : "Output from multiline grok {}", args : ["@{}"] } }
+
+{
+  convertTimestamp {
+    field : date
+    inputFormats : ["dd/MMM/yyyy HH:mm:ss Z"]
+    outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+    outputTimezone : UTC
+  }
+}

+ 3 - 8
desktop/libs/indexer/src/indexer/api3.py

@@ -49,14 +49,9 @@ def _escape_white_space_characters(s, inverse = False):
   return s
 
 def _convert_format(format_dict, inverse=False):
-  FIELDS = [
-    "fieldSeparator",
-    "quoteChar",
-    "recordSeparator"
-  ]
-
-  for field in FIELDS:
-    format_dict[field] = _escape_white_space_characters(format_dict[field], inverse)
+  for field in format_dict:
+    if isinstance(format_dict[field], basestring):
+      format_dict[field] = _escape_white_space_characters(format_dict[field], inverse)
 
 def guess_format(request):
   file_format = json.loads(request.POST.get('fileFormat', '{}'))

+ 2 - 2
desktop/libs/indexer/src/indexer/fields.py

@@ -49,8 +49,8 @@ class Field(object):
     'required': self.required}
 
 FIELD_TYPES = [
-  FieldType('text', "^.{100,}$"),
-  FieldType('string', "^.*$"),
+  FieldType('text_en', "^[\\s\\S]{100,}$"),
+  FieldType('string', "^[\\s\\S]*$"),
   FieldType('double', "^([+-]?[0-9]+\\.?[0-9]+)?$"),
   FieldType('long', "^(?:[+-]?(?:[0-9]+))?$"),
   FieldType('date', "^([0-9]+-[0-9]+-[0-9]+T[0-9]+:[0-9]+:[0-9]+(\\.[0-9]*)?Z)?$")

+ 39 - 7
desktop/libs/indexer/src/indexer/file_format.py

@@ -32,9 +32,20 @@ def _valid_csv_format(format_):
   return valid_has_header and valid_quote_char and valid_record_separator and valid_field_separator
 
 def get_file_format_instance(file_stream, format_=None):
-  if not format_ or _valid_csv_format(format_):
-    return CSVFormat(file_stream, format_)
-  return None
+  format_mapping = {
+    "csv": CSVFormat,
+    "hue": HueFormat
+  }
+
+  if format_ and "type" in format_:
+    type_ = format_["type"]
+  else:
+    type_ = "csv"
+
+  if type_ in format_mapping:
+    return format_mapping[type_](file_stream, format_)
+  else:
+    return None
 
 class FileFormat(object):
   def __init__(self):
@@ -72,6 +83,20 @@ class FileFormat(object):
 
     return obj
 
+class HueFormat(FileFormat):
+  def __init__(self, file_stream, format_):
+    self._fields = [
+      Field("date", "date"),
+      Field("component", "string"),
+      Field("log_level", "string"),
+      Field("details", "string"),
+      Field("message", "text")
+    ]
+
+  @property
+  def fields(self):
+    return self._fields
+
 class CSVFormat(FileFormat):
   def __init__(self, file_stream, format_=None):
     file_stream.seek(0)
@@ -84,10 +109,17 @@ class CSVFormat(FileFormat):
       self._quote_char = format_["quoteChar"].encode('utf-8')
       self._has_header = format_["hasHeader"]
     else:
-      dialect, self._has_header = self._guess_dialect(sample)
-      self._delimiter = dialect.delimiter
-      self._line_terminator = dialect.lineterminator
-      self._quote_char = dialect.quotechar
+      try:
+        dialect, self._has_header = self._guess_dialect(sample)
+        self._delimiter = dialect.delimiter
+        self._line_terminator = dialect.lineterminator
+        self._quote_char = dialect.quotechar
+      except Exception:
+        # guess dialect failed, fall back to defaults:
+        self._delimiter = ','
+        self._line_terminator = '\n'
+        self._quote_char = '"'
+        self._has_header = False
 
     # sniffer insists on \r\n even when \n. This is safer and good enough for a preview
     self._line_terminator = self._line_terminator.replace("\r\n", "\n")

+ 76 - 18
desktop/libs/indexer/src/indexer/templates/indexer.mako

@@ -57,16 +57,10 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
     <div data-bind="visible: createWizard.fileFormat().show">
       <div data-bind="with: createWizard.fileFormat().format">
-          <h3>${_('File Type')}: <span data-bind="text: type"></span></h3>
-          <h4>${_('Has Header')}:</h4>
-          <input type="checkbox" data-bind="checked: hasHeader">
-
-          <h4>${_('Quote Character')}:</h4>
-          <input data-bind="value: quoteChar">
-          <h4>${_('Record Separator')}:</h4>
-          <input data-bind="value: recordSeparator">
-          <h4>${_('Field Separator')}:</h4>
-          <input data-bind="value: fieldSeparator">
+          <h3>${_('File Type')}: <select data-bind="options: $root.createWizard.fileTypes, value: type"></select>
+          </h3>
+
+          <!-- ko template: {name: 'format-settings-'+type()}--><!-- /ko -->
         </div>
 
         <h3>${_('Fields')}</h3>
@@ -103,7 +97,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
         <h4 class="error" data-bind="visible: !createWizard.isNameAvailable() && createWizard.fileFormat().name().length == 0">${_('Collection needs a name')}</h4>
 
 
-        <a href="javascript:void(0)" class="btn btn-success" data-bind="visible: createWizard.jobId, attr: {           href: '/oozie/list_oozie_workflow/' + createWizard.jobId() }" target="_blank" title="${ _('Open') }">
+        <a href="javascript:void(0)" class="btn btn-success" data-bind="visible: createWizard.jobId, attr: {href: '/oozie/list_oozie_workflow/' + createWizard.jobId() }" target="_blank" title="${ _('Open') }">
           ${_('View Indexing Status')}
         </a>
 
@@ -113,6 +107,20 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
   </div>
 </script>
 
+<script type="text/html" id="format-settings-hue">
+</script>
+
+<script type="text/html" id="format-settings-csv">
+  <h4>${_('Has Header')}:</h4>
+  <input type="checkbox" data-bind="checked: hasHeader">
+  <h4>${_('Quote Character')}:</h4>
+  <input data-bind="value: quoteChar">
+  <h4>${_('Record Separator')}:</h4>
+  <input data-bind="value: recordSeparator">
+  <h4>${_('Field Separator')}:</h4>
+  <input data-bind="value: fieldSeparator">
+</script>
+
 <script type="text/html" id="field-template">
   <div>
     <span>${_('Keep')}</span><input type="checkbox" data-bind="checked: keep">
@@ -306,6 +314,54 @@ var getNewFieldName = function(){
     }
   }
 
+  var getFileFormat = function(type){
+    if(type == "csv"){
+      return new CsvFileType();
+    }
+    else if(type == "hue"){
+      return new HueFileType();
+    }
+  }
+
+  var FileType = function(){
+    var self = this;
+
+    self.loadFromObj = function(args){
+      for (var attr in args){
+        self[attr] = ko.mapping.fromJS(args[attr]);
+      }
+    }
+  }
+
+  var HueFileType = function(args){
+    var self = new FileType();
+
+    self.type = ko.observable("hue");
+
+    if(args) self.loadFromObj(args);
+
+    return self;
+  }
+
+  var CsvFileType = function(args){
+    var self = new FileType();
+
+    self.quoteChar = ko.observable('"');
+    self.recordSeparator = ko.observable("\\n");
+    self.type = ko.observable("csv");
+    self.hasHeader = ko.observable(false);
+    self.fieldSeparator = ko.observable(',');
+
+    if(args) self.loadFromObj(args);
+
+    self.quoteChar.subscribe(viewModel.createWizard.guessFieldTypes);
+    self.recordSeparator.subscribe(viewModel.createWizard.guessFieldTypes);
+    self.hasHeader.subscribe(viewModel.createWizard.guessFieldTypes);
+    self.fieldSeparator.subscribe(viewModel.createWizard.guessFieldTypes);
+
+    return self;
+  }
+
   var File_Format = function (vm) {
     var self = this;
 
@@ -325,6 +381,8 @@ var getNewFieldName = function(){
     self.operationTypes = ${operators_json | n};
 
     self.fieldTypes = ko.observableArray(${fields_json | n});
+    self.fileTypes = ["csv","hue"];
+
 
     self.show = ko.observable(true);
     self.showCreate = ko.observable(false);
@@ -349,13 +407,13 @@ var getNewFieldName = function(){
     });
 
     self.fileFormat().format.subscribe(function(){
-      self.fileFormat().format().quoteChar.subscribe(self.guessFieldTypes);
-      self.fileFormat().format().recordSeparator.subscribe(self.guessFieldTypes);
-      self.fileFormat().format().type.subscribe(self.guessFieldTypes);
-      self.fileFormat().format().hasHeader.subscribe(self.guessFieldTypes);
-      self.fileFormat().format().fieldSeparator.subscribe(self.guessFieldTypes);
-
       self.guessFieldTypes();
+
+      if(self.fileFormat().format().type){
+        self.fileFormat().format().type.subscribe(function(newType){
+          self.fileFormat().format(getFileFormat(newType));
+        });
+      }
     });
 
     self.guessFormat = function() {
@@ -364,7 +422,7 @@ var getNewFieldName = function(){
         "fileFormat": ko.mapping.toJSON(self.fileFormat)
       }, function(resp) {
 
-        self.fileFormat().format(ko.mapping.fromJS(resp));
+        self.fileFormat().format(new CsvFileType(resp));
 
         self.fileFormat().show(true);