فهرست منبع

HUE-4341 [indexer] Use info level logging for morphlines, add error handling for invalid dates

Aaron Peddle 9 سال پیش
والد
کامیت
7af37c1

+ 19 - 8
desktop/libs/indexer/src/data/oozie_workspace/clean_to_match_schema.conf

@@ -51,7 +51,7 @@
           { equals { "${field['name']}" : [] } }
         ]
         then : [
-          { logTrace { format : "Ignoring record because it has no ${field['name']}: {}", args : ["@{}"] } }
+          { logInfo { format : "Ignoring record because it has no ${field['name']}: {}", args : ["@{}"] } }
           { dropRecord {} }
         ]
       }
@@ -63,10 +63,21 @@
 % for field in fields:
   %if field["type"] == 'date':
     {
-      convertTimestamp {
-        field : ${field['name']}
-        outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
-        outputTimezone : UTC
+      if {
+        conditions : [
+          {
+            convertTimestamp {
+              field : ${field['name']}
+              outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+              outputTimezone : UTC
+            }
+          }
+        ]
+        then : []
+        else : [
+          { logInfo { format : "Ignoring record due to invalid date for ${field['name']}: {}", args : ["@{}"] } }
+          { dropRecord {} }
+        ]
       }
     }
   %endif
@@ -99,13 +110,13 @@
                 ]
                 then : []
                 else : [
-                  { logTrace { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
+                  { logInfo { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
                   { dropRecord {} }
                 ]
               }
             }
           %else:
-            { logTrace { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
+            { logInfo { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
             { dropRecord {} }
           %endif
         ]
@@ -120,4 +131,4 @@
     # Location from which to fetch Solr schema
     solrLocator : <%text>${SOLR_LOCATOR}</%text>
   }
-}
+}

+ 1 - 1
desktop/libs/indexer/src/data/oozie_workspace/log4j.properties

@@ -20,7 +20,7 @@ log4j.logger.org.apache.flume.sink=INFO
 #log4j.logger.org.apache.flume.sink.solr=DEBUG
 log4j.logger.org.apache.solr=INFO
 #log4j.logger.org.apache.solr.hadoop=DEBUG
-log4j.logger.org.kitesdk.morphline=TRACE
+log4j.logger.org.kitesdk.morphline=INFO
 #log4j.logger.org.apache.solr.morphline=DEBUG
 log4j.logger.org.apache.solr.update.processor.LogUpdateProcessor=WARN
 log4j.logger.org.apache.solr.core.SolrCore=WARN