Browse Source

HUE-8208 [importer] Automatically select Kudu as output table format when Kafka input

Romain Rigaux 7 years ago
parent
commit
fa34e9bae6

+ 4 - 5
desktop/libs/indexer/src/indexer/indexers/envelope.py

@@ -111,9 +111,7 @@ steps {
         deriver {
             type = sql
             query.literal = \"""
-                SELECT UNIX_TIMESTAMP() * 1000 as_of_time, ROUND(AVG(number_of_vehicles), 2) avg_num_veh,
-                MIN(number_of_vehicles) min_num_veh, MAX(number_of_vehicles) max_num_veh,
-                MIN(measurement_time) first_meas_time, MAX(measurement_time) last_meas_time FROM traffic\"""
+                SELECT measurement_time, number_of_vehicles FROM traffic\"""
         }
         planner {
             type = upsert
@@ -129,7 +127,8 @@ steps {
 """ % properties
 
 
-"""application {
+  def generate_config_parquet(self, properties):
+    return """application {
   name = Filesystem Example
   executors = 1
 }
@@ -159,4 +158,4 @@ steps {
     }
   }
 }
-"""
+"""

+ 17 - 10
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -1291,6 +1291,7 @@ ${ assist.assistPanel() }
         self.rdbmsMode('customRdbms');
         if (val == 'kafka') {
           wizard.guessFormat();
+          wizard.destination.tableFormat('kudu');
         }
       });
       self.inputFormatsAll = ko.observableArray([
@@ -1769,16 +1770,22 @@ ${ assist.assistPanel() }
       self.KUDU_DEFAULT_RANGE_PARTITION_COLUMN = {values: [{value: ''}], name: 'VALUES', lower_val: 0, include_lower_val: '<=', upper_val: 1, include_upper_val: '<='};
       self.KUDU_DEFAULT_PARTITION_COLUMN = {columns: [], range_partitions: [self.KUDU_DEFAULT_RANGE_PARTITION_COLUMN], name: 'HASH', int_val: 16};
 
-      self.tableFormats = ko.observableArray([
-          {'value': 'text', 'name': 'Text'},
-          {'value': 'parquet', 'name': 'Parquet'},
-          {'value': 'kudu', 'name': 'Kudu'},
-          {'value': 'csv', 'name': 'Csv'},
-          {'value': 'avro', 'name': 'Avro'},
-          {'value': 'json', 'name': 'Json'},
-          {'value': 'regexp', 'name': 'Regexp'},
-          {'value': 'orc', 'name': 'ORC'},
-      ]);
+      self.tableFormats = ko.pureComputed(function() {
+        if (wizard.source.inputFormat() == 'kafka') {
+          return [{'value': 'kudu', 'name': 'Kudu'}];
+        } else {
+          return [
+            {'value': 'text', 'name': 'Text'},
+            {'value': 'parquet', 'name': 'Parquet'},
+            {'value': 'kudu', 'name': 'Kudu'},
+            {'value': 'csv', 'name': 'Csv'},
+            {'value': 'avro', 'name': 'Avro'},
+            {'value': 'json', 'name': 'Json'},
+            {'value': 'regexp', 'name': 'Regexp'},
+            {'value': 'orc', 'name': 'ORC'},
+          ]
+        }
+      });
 
       self.partitionColumns = ko.observableArray();
       self.kuduPartitionColumns = ko.observableArray();