Browse Source

HUE-7345 [indexer] Accept x01 char as field separator

Properly filter out fields with NULL values instead of disgarding the full record.
Romain Rigaux 8 years ago
parent
commit
3fc2938a66

+ 1 - 1
desktop/libs/indexer/src/data/oozie_workspace/clean_to_match_schema.conf

@@ -24,7 +24,7 @@
           {
           {
             grok {
             grok {
               expressions : {
               expressions : {
-                "${field['name']}" : "^$"
+                "${field['name']}" : "^()|(NULL)|(\\\\N)$"
               }
               }
               extract : false
               extract : false
             }
             }

+ 2 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -214,7 +214,7 @@ def _create_index(user, fs, client, source, destination, index_name):
   df = destination['indexerPrimaryKey'] and destination['indexerPrimaryKey'][0] or None
   df = destination['indexerPrimaryKey'] and destination['indexerPrimaryKey'][0] or None
   kwargs = {}
   kwargs = {}
 
 
-  if source['inputFormat'] != 'manual':
+  if source['inputFormat'] not in ('manual', 'table'):
     stats = fs.stats(source["path"])
     stats = fs.stats(source["path"])
     if stats.size > MAX_UPLOAD_SIZE:
     if stats.size > MAX_UPLOAD_SIZE:
       raise PopupException(_('File size is too large to handle!'))
       raise PopupException(_('File size is too large to handle!'))
@@ -248,7 +248,7 @@ def _create_index(user, fs, client, source, destination, index_name):
         replication=destination['indexerReplicationFactor']
         replication=destination['indexerReplicationFactor']
     )
     )
 
 
-  if source['inputFormat'] != 'manual':
+  if source['inputFormat'] not in ('manual', 'table'):
     data = fs.read(source["path"], 0, MAX_UPLOAD_SIZE)
     data = fs.read(source["path"], 0, MAX_UPLOAD_SIZE)
     client.index(name=index_name, data=data, **kwargs)
     client.index(name=index_name, data=data, **kwargs)
 
 

+ 1 - 0
desktop/libs/indexer/src/indexer/file_format.py

@@ -321,6 +321,7 @@ class CSVFormat(FileFormat):
     string = string.replace('\t', '\\t')
     string = string.replace('\t', '\\t')
     string = string.replace('\n', '\\n')
     string = string.replace('\n', '\\n')
     string = string.replace('\u0001', '\\u0001')
     string = string.replace('\u0001', '\\u0001')
+    string = string.replace('\x01', '\\u0001')
 
 
     return string
     return string
 
 

+ 19 - 12
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -1193,7 +1193,11 @@ ${ assist.assistPanel() }
         }
         }
 
 
         for (var i = 0; i < type.args.length; i++) {
         for (var i = 0; i < type.args.length; i++) {
-          self[type.args[i].name].subscribe(viewModel.createWizard.guessFieldTypes);
+          self[type.args[i].name].subscribe(function(newVal) {
+            if (newVal) { // Double call on non File selection otherwise
+              viewModel.createWizard.guessFieldTypes();
+            }
+          });
         }
         }
       }
       }
 
 
@@ -1257,8 +1261,8 @@ ${ assist.assistPanel() }
       self.path = ko.observable('');
       self.path = ko.observable('');
       self.path.subscribe(function(val) {
       self.path.subscribe(function(val) {
         if (val) {
         if (val) {
-          vm.createWizard.guessFormat();
-          vm.createWizard.destination.nonDefaultLocation(val);
+          wizard.guessFormat();
+          wizard.destination.nonDefaultLocation(val);
         }
         }
         resizeElements();
         resizeElements();
       });
       });
@@ -1266,7 +1270,7 @@ ${ assist.assistPanel() }
         return self.inputFormat() == 'file' && /^(s3a|adl):\/.*$/.test(self.path());
         return self.inputFormat() == 'file' && /^(s3a|adl):\/.*$/.test(self.path());
       });
       });
       self.isObjectStore.subscribe(function(newVal) {
       self.isObjectStore.subscribe(function(newVal) {
-        vm.createWizard.destination.useDefaultLocation(!newVal);
+        wizard.destination.useDefaultLocation(!newVal);
       });
       });
       // Rdbms
       // Rdbms
       self.rdbmsMode = ko.observable('customRdbms');
       self.rdbmsMode = ko.observable('customRdbms');
@@ -1413,8 +1417,8 @@ ${ assist.assistPanel() }
       });
       });
       self.table.subscribe(function(val) {
       self.table.subscribe(function(val) {
         if (val) {
         if (val) {
-          vm.createWizard.guessFormat();
-          vm.createWizard.destination.nonDefaultLocation(val);
+          wizard.guessFormat();
+          wizard.destination.nonDefaultLocation(val);
         }
         }
         resizeElements();
         resizeElements();
       });
       });
@@ -1427,17 +1431,19 @@ ${ assist.assistPanel() }
       self.format = ko.observable();
       self.format = ko.observable();
       self.format.subscribe(function(newVal) {
       self.format.subscribe(function(newVal) {
         if (typeof newVal.hasHeader !== 'undefined') {
         if (typeof newVal.hasHeader !== 'undefined') {
-          vm.createWizard.destination.hasHeader(newVal.hasHeader());
+          wizard.destination.hasHeader(newVal.hasHeader());
           newVal.hasHeader.subscribe(function(newVal) {
           newVal.hasHeader.subscribe(function(newVal) {
-            vm.createWizard.destination.hasHeader(newVal);
+            wizard.destination.hasHeader(newVal);
           });
           });
         }
         }
 
 
         if (typeof newVal.fieldSeparator !== 'undefined') {
         if (typeof newVal.fieldSeparator !== 'undefined') {
-          vm.createWizard.destination.useCustomDelimiters(newVal.fieldSeparator() != ',');
-          vm.createWizard.destination.customFieldDelimiter(newVal.fieldSeparator());
+          wizard.destination.useCustomDelimiters(newVal.fieldSeparator() != ',');
+          wizard.destination.customFieldDelimiter(newVal.fieldSeparator());
           newVal.fieldSeparator.subscribe(function(newVal) {
           newVal.fieldSeparator.subscribe(function(newVal) {
-            vm.createWizard.destination.customFieldDelimiter(newVal);
+            if (newVal != '') {
+              wizard.destination.customFieldDelimiter(newVal);
+            }
           });
           });
         }
         }
       });
       });
@@ -1794,6 +1800,7 @@ ${ assist.assistPanel() }
         {'value': '\\001', 'name': '${ _("^A (\\001)") }'},
         {'value': '\\001', 'name': '${ _("^A (\\001)") }'},
         {'value': '\\002', 'name': '${ _("^B (\\002)") }'},
         {'value': '\\002', 'name': '${ _("^B (\\002)") }'},
         {'value': '\\003', 'name': '${ _("^C (\\003)") }'},
         {'value': '\\003', 'name': '${ _("^C (\\003)") }'},
+        {'value': '\x01', 'name': '${ _("^A (\\x01)") }'}
       ]);
       ]);
 
 
       self.editorId = ko.observable();
       self.editorId = ko.observable();
@@ -1842,7 +1849,7 @@ ${ assist.assistPanel() }
         }
         }
 
 
         if (self.source.format().type) {
         if (self.source.format().type) {
-          if (!self.formatTypeSubscribed) {
+          if (! self.formatTypeSubscribed) {
             self.formatTypeSubscribed = true;
             self.formatTypeSubscribed = true;
             self.source.format().type.subscribe(function (newType) {
             self.source.format().type.subscribe(function (newType) {
               self.source.format(new FileType(newType));
               self.source.format(new FileType(newType));