فهرست منبع

HUE-4471 [indexer] Add geoip operation automatically to hue format

Aaron Peddle 9 سال پیش
والد
کامیت
c70e6c3

+ 38 - 18
desktop/libs/indexer/src/data/oozie_workspace/geo_ip_operation.conf

@@ -15,25 +15,45 @@
 # limitations under the License.from nose.tools import assert_equal
 <%page args="field, operation"/>
 
-# field
-# operation
 {
-  geoIP {
-    inputField : "${field['name']}"
-    database : "GeoLite2-City.mmdb"
-  }
-}
+  if{
+    conditions : [
+      # check if the IP is reserved
+      {
+        grok {
+          dictionaryFiles : ["grok_dictionaries"]
 
-# extract parts of the geolocation info from the Jackson JsonNode Java
-# object contained in the _attachment_body field and store the parts in
-# the given record output fields:
-{
-  extractJsonPaths {
-    flatten : false
-    paths : {
-      %for i, component in enumerate(get_kept_args(operation)):
-        "${operation['fields'][i]['name']}" : "${component.name}"
-      %endfor
-    }
+          expressions : {
+            ${field['name']}: """((^127\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)).*"""
+          }
+        }
+      }
+    ]
+    then : [
+    ]
+    else : [
+      # field
+      # operation
+      {
+        geoIP {
+          inputField : "${field['name']}"
+          database : "GeoLite2-City.mmdb"
+        }
+      }
+
+      # extract parts of the geolocation info from the Jackson JsonNode Java
+      # object contained in the _attachment_body field and store the parts in
+      # the given record output fields:
+      {
+        extractJsonPaths {
+          flatten : false
+          paths : {
+            %for i, component in enumerate(get_kept_args(operation)):
+              "${operation['fields'][i]['name']}" : "${component.name}"
+            %endfor
+          }
+        }
+      }
+    ]
   }
 }

+ 14 - 2
desktop/libs/indexer/src/data/oozie_workspace/morphline_template.conf

@@ -55,7 +55,19 @@ morphlines : [
       # process operations on fields
       % for field in fields:
         %for operation in field['operations']:
-          <%include file="${operation['type']}_operation.conf" args="field=field, operation=operation"/>
+          {
+            if {
+              conditions : [
+                { equals { "${field['name']}" : [] } }
+              ]
+              then : [
+                # skip the operation as the required field isn't present
+              ]
+              else : [
+                <%include file="${operation['type']}_operation.conf" args="field=field, operation=operation"/>
+              ]
+            }
+          }
         %endfor
       %endfor
 
@@ -72,4 +84,4 @@ morphlines : [
 
     ]
   }
-]
+]

+ 26 - 24
desktop/libs/indexer/src/data/oozie_workspace/parse_hue.conf

@@ -25,40 +25,42 @@
 { logTrace { format : "Output from multiline {}", args : ["@{}"] } }
 
 {
-  tryRules {
-    catchExceptions : true
-    throwExceptionIfAllRulesFailed : true
-    rules : [
-      # next rule of tryRules cmd:
+  if{
+    conditions : [
+      # check if the first grok will work (matches info) and if it does then do it. Actions in conditions don't persist
       {
-        commands : [
-          {
-            grok {
-              dictionaryFiles : ["grok_dictionaries"]
+        grok {
+          dictionaryFiles : ["grok_dictionaries"]
 
-              expressions : {
-                message: """\[%{GREEDYDATA:date}\] %{WORD:component} +%{LOGLEVEL:log_level} +%{IP:ip} %{WORD:user} - "%{WORD:http_method} %{NOTSPACE:path} %{NOTSPACE:protocol}"%{GREEDYDATA:details}"""
-              }
-            }
+          expressions : {
+            message: """\[%{GREEDYDATA:date}\] %{WORD:component} +%{LOGLEVEL:log_level} +%{IP:ip} %{WORD:user} - "%{WORD:http_method} %{NOTSPACE:path} %{NOTSPACE:protocol}"%{GREEDYDATA:details}"""
           }
-        ]
+        }
       }
+    ]
+    then : [
+      {
+        grok {
+          dictionaryFiles : ["grok_dictionaries"]
 
-      # next rule of tryRules cmd:
+          expressions : {
+            message: """\[%{GREEDYDATA:date}\] %{WORD:component} +%{LOGLEVEL:log_level} +%{IP:ip} %{WORD:user} - "%{WORD:http_method} %{NOTSPACE:path} %{NOTSPACE:protocol}"%{GREEDYDATA:details}"""
+          }
+        }
+      }
+    ]
+    else : [
       {
-        commands : [
-          {
-            grok {
-              dictionaryFiles : ["grok_dictionaries"]
+        grok {
+          dictionaryFiles : ["grok_dictionaries"]
 
-              expressions : {
-                message: """\[%{GREEDYDATA:date}\] %{WORD:component} +%{LOGLEVEL:log_level} %{GREEDYDATA:details}[\s\S]*"""
-              }
-            }
+          expressions : {
+            message: """\[%{GREEDYDATA:date}\] %{WORD:component} +%{LOGLEVEL:log_level} %{GREEDYDATA:details}[\s\S]*"""
           }
-        ]
+        }
       }
     ]
   }
 }
+
 { logTrace { format : "Output from multiline grok {}", args : ["@{}"] } }

+ 1 - 1
desktop/libs/indexer/src/indexer/fields.py

@@ -50,7 +50,7 @@ class Field(object):
     return {'name': self.name,
     'type': self.field_type_name,
     'keep': self.keep,
-    'operations': [operation.to_dict() for operation in self.operations],
+    'operations': self.operations,
     'required': self.required}
 
 FIELD_TYPES = [

+ 16 - 1
desktop/libs/indexer/src/indexer/file_format.py

@@ -22,6 +22,7 @@ from django.utils.translation import ugettext as _
 
 from indexer.fields import Field, guess_field_type_from_samples
 from indexer.argument import TextArgument, CheckboxArgument
+from indexer.operations import get_operator
 
 LOG = logging.getLogger(__name__)
 
@@ -161,13 +162,27 @@ class HueLogFormat(GrokkedFormat):
   _extensions = ["log"]
 
   def __init__(self):
+    geo_ip_operation = get_operator("geo_ip").get_default_operation()
+
+    geo_ip_operation['settings']["/country/names/en"] = True
+    geo_ip_operation['settings']["/city/names/en"] = True
+    geo_ip_operation['settings']["/location/latitude"] = True
+    geo_ip_operation['settings']["/location/longitude"] = True
+
+    geo_ip_operation['fields'] += [
+      Field("country", "string").to_dict(),
+      Field("city", "string").to_dict(),
+      Field("latitude", "double").to_dict(),
+      Field("longitude", "double").to_dict()
+    ]
+
     self._fields = [
       Field("date", "date"),
       Field("component", "string"),
       Field("log_level", "string"),
       Field("details", "string"),
       Field("message", "text_en"),
-      Field("ip", "string"),
+      Field("ip", "string", [geo_ip_operation]),
       Field("user", "string"),
       Field("http_method", "string"),
       Field("path", "string"),