瀏覽代碼

HUE-4534 [indexer] Fix syslog groking for smart indexer

peddle 9 年之前
父節點
當前提交
cf1532d

+ 1 - 3
desktop/libs/indexer/src/data/oozie_workspace/clean_to_match_schema.conf

@@ -78,9 +78,7 @@
                     field : ${field['name']}
                     outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
                     outputTimezone : UTC
-                  }
-                }
-              ]
+                  } } ]
               then : []
               else : [
               # the built in convertTimestamp options failed, try some custom ones

+ 2 - 2
desktop/libs/indexer/src/data/oozie_workspace/parse_grok_line.conf

@@ -42,7 +42,7 @@
           dictionaryFiles : ["grok_dictionaries"]
 
           expressions : {
-            input_line: """${format_class.get_grok()}"""
+            file_line: """${format_class.get_grok()}"""
           }
         }
       }
@@ -53,7 +53,7 @@
           dictionaryFiles : ["grok_dictionaries"]
 
           expressions : {
-            input_line: """${format_class.get_grok()}"""
+            file_line: """${format_class.get_grok()}"""
           }
         }
       }

+ 23 - 15
desktop/libs/indexer/src/indexer/file_format.py

@@ -31,7 +31,8 @@ def get_format_types():
     CSVFormat,
     HueLogFormat,
     ApacheCombinedFormat,
-    RubyLogFormat
+    RubyLogFormat,
+    SyslogFormat
   ]
 
 def _get_format_mapping():
@@ -141,6 +142,7 @@ class FileFormat(object):
 
 class GrokkedFormat(FileFormat):
   _grok = None
+  _customizable = False
 
   @classmethod
   def get_grok(cls):
@@ -155,10 +157,13 @@ class GrokkedFormat(FileFormat):
 
     return format_
 
+  @property
+  def fields(self):
+    return self._fields
+
 class HueLogFormat(GrokkedFormat):
   _name = "hue"
   _description = _("Hue Log File")
-  _customizable = False
   _extensions = ["log"]
 
   def __init__(self):
@@ -189,17 +194,12 @@ class HueLogFormat(GrokkedFormat):
       Field("protocol", "string")
     ]
 
-  @property
-  def fields(self):
-    return self._fields
-
 class GrokLineFormat(GrokkedFormat):
   _parse_type = "grok_line"
 
 class ApacheCombinedFormat(GrokLineFormat):
   _name = "combined_apache"
   _description = _("Combined Apache Log File")
-  _customizable = False
   _extensions = ["log"]
   _grok = "%{COMBINEDAPACHELOG}"
 
@@ -219,14 +219,9 @@ class ApacheCombinedFormat(GrokLineFormat):
       Field("field_line", "text_en")
     ]
 
-  @property
-  def fields(self):
-    return self._fields
-
 class RubyLogFormat(GrokLineFormat):
   _name = "ruby_log"
   _description = _("Ruby Log")
-  _customizable = False
   _extensions = ["log"]
   _grok = "%{RUBY_LOGGER}"
 
@@ -240,9 +235,22 @@ class RubyLogFormat(GrokLineFormat):
       Field("field_line", "text_en")
     ]
 
-  @property
-  def fields(self):
-    return self._fields
+class SyslogFormat(GrokLineFormat):
+  _name = "syslog"
+  _description = _("Syslog")
+  _grok = "%{SYSLOGLINE}"
+
+  def __init__(self):
+    self._fields = [
+      Field("timestamp", "string"),
+      Field("timestamp8601", "string"),
+      Field("facility", "string"),
+      Field("priority", "string"),
+      Field("logsource", "string"),
+      Field("program", "string"),
+      Field("pid", "string"),
+      Field("message", "text_en"),
+    ]
 
 class CSVFormat(FileFormat):
   _name = "csv"