瀏覽代碼

HUE-4384 [indexer] Add support for combined apache log files

Aaron Peddle 9 年之前
父節點
當前提交
1e49e73

+ 60 - 0
desktop/libs/indexer/src/data/oozie_workspace/parse_combined_apache.conf

@@ -0,0 +1,60 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from nose.tools import assert_equal
+<%page args="format, format_character"/>
+
+{
+  readLine{
+    charset: UTF-8
+  }
+}
+
+
+{
+  tryRules {
+    catchExceptions : true
+    throwExceptionIfAllRulesFailed : false
+    rules : [
+      {
+        commands : [
+          {
+            grok {
+              dictionaryFiles : ["grok_dictionaries"]
+
+              expressions : {
+                message: """%{COMBINEDAPACHELOG}"""
+              }
+            }
+          }
+        ]
+      }
+      {
+        # do nothing if match of grok failed
+        commands : [
+          { logInfo { format : "failed to grok line {}", args : ["@{}"] } }
+        ]
+      }
+    ]
+  }
+}
+
+{
+  convertTimestamp {
+    field : timestamp
+    inputFormats : ["dd/MMM/yyyy:HH:mm:ss Z"]
+    outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+    outputTimezone : UTC
+  }
+}

+ 28 - 1
desktop/libs/indexer/src/indexer/file_format.py

@@ -28,7 +28,8 @@ LOG = logging.getLogger(__name__)
 def get_format_types():
   return [
     CSVFormat,
-    HueFormat
+    HueFormat,
+    ApacheFormat
   ]
 
 def get_format_mapping():
@@ -147,6 +148,32 @@ class HueFormat(FileFormat):
   def fields(self):
     return self._fields
 
+class ApacheFormat(FileFormat):
+  _name = "combined_apache"
+  _description = _("Combined Apache Log File")
+  _customizable = False
+  _extensions = ["log"]
+
+  def __init__(self, file_stream, format_):
+    self._fields = [
+      Field("clientip", "string"),
+      Field("ident", "string"),
+      Field("auth", "string"),
+      Field("timestamp", "date"),
+      Field("verb", "string"),
+      Field("request", "string"),
+      Field("httpversion", "double"),
+      Field("rawrequest", "long"),
+      Field("response", "long"),
+      Field("bytes", "long"),
+      Field("referrer", "string"),
+      Field("message", "text_en")
+    ]
+
+  @property
+  def fields(self):
+    return self._fields
+
 class CSVFormat(FileFormat):
   _name = "csv"
   _description = _("CSV File")