Эх сурвалжийг харах

HUE-4527 [indexer] Date fields that aren't required can crash the indexing job

peddle 9 жил өмнө
parent
commit
e4c3a81

+ 24 - 13
desktop/libs/indexer/src/data/oozie_workspace/clean_to_match_schema.conf

@@ -63,26 +63,19 @@
 % for field in fields:
   %if field["type"] == 'date':
     {
+      # check that the field exists before running the operation
       if {
         conditions : [
-          {
-            convertTimestamp {
-              field : ${field['name']}
-              outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
-              outputTimezone : UTC
-            }
-          }
+            { equals { "${field['name']}" : [] } }
         ]
         then : []
         else : [
-        # the built in convertTimestamp options failed, try some custom ones
           {
             if {
               conditions : [
                 {
                   convertTimestamp {
-                    field : date
-                    inputFormats : ["dd/MMM/yyyy HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss Z"]
+                    field : ${field['name']}
                     outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
                     outputTimezone : UTC
                   }
@@ -90,9 +83,27 @@
               ]
               then : []
               else : [
-                # none of the custom dates matched either. Drop the record
-                { logInfo { format : "Ignoring record due to invalid date for ${field['name']}: {}", args : ["@{}"] } }
-                { dropRecord {} }
+              # the built in convertTimestamp options failed, try some custom ones
+                {
+                  if {
+                    conditions : [
+                      {
+                        convertTimestamp {
+                          field : date
+                          inputFormats : ["dd/MMM/yyyy HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss Z"]
+                          outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+                          outputTimezone : UTC
+                        }
+                      }
+                    ]
+                    then : []
+                    else : [
+                      # none of the custom dates matched either. Drop the record
+                      { logInfo { format : "Ignoring record due to invalid date for ${field['name']}: {}", args : ["@{}"] } }
+                      { dropRecord {} }
+                    ]
+                  }
+                }
               ]
             }
           }