浏览代码

HUE-4641 [indexer] GeoIP will stop indexing if an IP is missing from it's lookup table

peddle 9 年之前
父节点
当前提交
27b50558b1
共有 1 个文件被更改,包括 39 次插入19 次删除
  1. 39 19
      desktop/libs/indexer/src/data/oozie_workspace/geo_ip_operation.conf

+ 39 - 19
desktop/libs/indexer/src/data/oozie_workspace/geo_ip_operation.conf

@@ -24,36 +24,56 @@
           dictionaryFiles : ["grok_dictionaries"]
 
           expressions : {
+            # Check if the ip is private. And skip if it is
             ${field['name']}: """((^127\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)).*"""
           }
         }
       }
     ]
     then : [
+      { logDebug { format : "GeoIP failed - this IP is reserved: {}", args : ["@{}"] } }
     ]
     else : [
-      # field
-      # operation
-      {
-        geoIP {
-          inputField : "${field['name']}"
-          database : "GeoLite2-City.mmdb"
-        }
-      }
+    {
+      tryRules {
+        catchExceptions : true
+        throwExceptionIfAllRulesFailed : true
+        rules : [
+          # try looking up the ip:
+          {
+            commands : [
+                    {
+                      geoIP {
+                        inputField : "${field['name']}"
+                        database : "GeoLite2-City.mmdb"
+                      }
+                    }
 
-      # extract parts of the geolocation info from the Jackson JsonNode Java
-      # object contained in the _attachment_body field and store the parts in
-      # the given record output fields:
-      {
-        extractJsonPaths {
-          flatten : false
-          paths : {
-            %for i, component in enumerate(get_kept_args(operation)):
-              "${operation['fields'][i]['name']}" : "${component.name}"
-            %endfor
+                    # extract parts of the geolocation info from the Jackson JsonNode Java
+                    # object contained in the _attachment_body field and store the parts in
+                    # the given record output fields:
+                    {
+                      extractJsonPaths {
+                        flatten : false
+                        paths : {
+                          %for i, component in enumerate(get_kept_args(operation)):
+                            "${operation['fields'][i]['name']}" : "${component.name}"
+                          %endfor
+                        }
+                      }
+                    }
+            ]
           }
-        }
+
+          # lookup failed fallback to leaving fields empty:
+          {
+            commands : [
+              { logDebug { format : "GeoIP lookup failed: {}", args : ["@{}"] } }
+            ]
+          }
+        ]
       }
+    }
     ]
   }
 }