Browse Source

HUE-8208 [importer] Combine the stream inputs into one

Romain Rigaux 7 years ago
parent
commit
8f2ae75942

+ 1 - 1
desktop/core/src/desktop/templates/hue.mako

@@ -224,7 +224,7 @@ ${ hueIcons.symbols() }
 
 
       <div class="top-nav-middle">
       <div class="top-nav-middle">
 
 
-        <!-- ko if: cluster.clusters().length > 1 && cluster.clusters()[0].type() != ANALYTIC_DB -->
+        <!-- ko if: cluster.clusters().length > 1 && cluster.clusters()[0].type() != "analyticdbs" -->
         <div class="btn-group pull-right" style="display: none;" data-bind="visible: cluster.clusters().length > 1">
         <div class="btn-group pull-right" style="display: none;" data-bind="visible: cluster.clusters().length > 1">
           <button class="btn" data-bind="text: cluster.cluster().name() + (cluster.cluster().interface ? ' ' + cluster.cluster().interface() : '')"></button>
           <button class="btn" data-bind="text: cluster.cluster().name() + (cluster.cluster().interface ? ' ' + cluster.cluster().interface() : '')"></button>
           <button class="btn dropdown-toggle" data-toggle="dropdown">
           <button class="btn dropdown-toggle" data-toggle="dropdown">

File diff suppressed because it is too large
+ 6 - 4
desktop/libs/indexer/src/indexer/api3.py


+ 6 - 6
desktop/libs/indexer/src/indexer/indexers/envelope.py

@@ -94,16 +94,16 @@ class EnvelopeIndexer(object):
               }
               }
       """ % properties
       """ % properties
     elif properties['inputFormat'] == 'streams':
     elif properties['inputFormat'] == 'streams':
-      if properties['publicStreamsSelection'] == 'SFDC':
+      if properties['streamSelection'] == 'SFDC':
         input = """      type = sfdc
         input = """      type = sfdc
         mode = fetch-all
         mode = fetch-all
-        sobject = %(publicStreamsObject)s
+        sobject = %(streamObject)s
         sfdc: {
         sfdc: {
           partner: {
           partner: {
-            username = "%(publicStreamsUsername)s"
-            password = "%(publicStreamsPassword)s"
-            token = "%(publicStreamsToken)s"
-            auth-endpoint = "%(publicStreamsEndpointUrl)s"
+            username = "%(streamUsername)s"
+            password = "%(streamPassword)s"
+            token = "%(streamToken)s"
+            auth-endpoint = "%(streamEndpointUrl)s"
           }
           }
         }
         }
   """
   """

+ 72 - 69
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -281,7 +281,6 @@ ${ assist.assistPanel() }
                 </label>
                 </label>
               </div>
               </div>
 
 
-
               <div class="control-group">
               <div class="control-group">
                 <button class="btn" data-bind="click: createWizard.source.rdbmsCheckConnection">
                 <button class="btn" data-bind="click: createWizard.source.rdbmsCheckConnection">
                   ${_('Test Connection')}
                   ${_('Test Connection')}
@@ -322,66 +321,66 @@ ${ assist.assistPanel() }
 
 
           <!-- /ko -->
           <!-- /ko -->
 
 
-          <!-- ko if: createWizard.source.inputFormat() == 'kafka' -->
+          <!-- ko if: createWizard.source.inputFormat() == 'stream' -->
             <div class="control-group">
             <div class="control-group">
-              <label class="control-label"><div>${ _('Topics') }</div>
-                <select data-bind="options: createWizard.source.kafkaTopics,
-                       value: createWizard.source.kafkaSelectedTopics,
-                       optionsCaption: 'Choose...'"
-                       placeholder="${ _('The list of topics to consume, e.g. orders,returns') }"></select>
-                ## <select data-bind="selectize: createWizard.source.kafkaTopics, value: createWizard.source.kafkaSelectedTopics" placeholder="${ _('The list of topics to consume, e.g. orders,returns') }"></select>
+              <label class="control-label"><div>${ _('List') }</div>
+                <select data-bind="selectize: createWizard.source.publicStreams, value: createWizard.source.streamSelection, optionsText: 'name', optionsValue: 'value'" placeholder="${ _('The list of streams to consume, e.g. SFDC, Jiras...') }"></select>
               </label>
               </label>
-              
-              <br/>
-              
-              <span data-bind="visible: createWizard.source.kafkaSelectedTopics">
-                <label class="control-label"><div>${ _('Field names') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldNames" placeholder="${ _('The list of fields to consume, e.g. orders,returns') }">
-                </label>
+            </div>
 
 
-                <label class="control-label"><div>${ _('Field types') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldTypes" placeholder="${ _('The list of topics to consume, e.g. orders,returns') }">
+            <!-- ko if: createWizard.source.streamSelection() == 'kafka' -->
+              <div class="control-group">
+                <label class="control-label"><div>${ _('Topics') }</div>
+                  <select data-bind="options: createWizard.source.kafkaTopics,
+                         value: createWizard.source.kafkaSelectedTopics,
+                         optionsCaption: 'Choose...'"
+                         placeholder="${ _('The list of topics to consume, e.g. orders,returns') }"></select>
+                  ## <select data-bind="selectize: createWizard.source.kafkaTopics, value: createWizard.source.kafkaSelectedTopics" placeholder="${ _('The list of topics to consume, e.g. orders,returns') }"></select>
                 </label>
                 </label>
-              </span>
-            </div>
-          <!-- /ko -->
 
 
-          <!-- ko if: createWizard.source.inputFormat() == 'streams' -->
-            <div class="control-group">
-              <label class="control-label"><div>${ _('List') }</div>
-                <select data-bind="selectize: createWizard.source.publicStreams, value: createWizard.source.publicStreamsSelection" placeholder="${ _('The list of streams to consume, e.g. SFDC, Jiras...') }"></select>
-              </label>
-            </div>
+                <br/>
+
+##                <span data-bind="visible: createWizard.source.kafkaSelectedTopics">
+##          <label class="control-label"><div>${ _('Field names') }</div>
+##                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldNames" placeholder="${ _('The list of fields to consume, e.g. orders,returns') }">
+##                </label>
 
 
-            <!-- ko if: createWizard.source.publicStreamsSelection() == 'SFDC' -->
+##                <label class="control-label"><div>${ _('Field types') }</div>
+##                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldTypes" placeholder="${ _('The list of topics to consume, e.g. orders,returns') }">
+##                </label>
+##              </span>
+              </div>
+            <!-- /ko -->
+
+            <!-- ko if: createWizard.source.streamSelection() == 'sfdc' -->
               <div class="control-group">
               <div class="control-group">
                 <label class="control-label"><div>${ _('Username') }</div>
                 <label class="control-label"><div>${ _('Username') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.publicStreamsUsername">
+                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamUsername">
                 </label>
                 </label>
-  
+
                 <label class="control-label"><div>${ _('Password') }</div>
                 <label class="control-label"><div>${ _('Password') }</div>
-                  <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.publicStreamsPassword">
+                  <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.streamPassword">
                 </label>
                 </label>
-  
+
                 <label class="control-label"><div>${ _('Token') }</div>
                 <label class="control-label"><div>${ _('Token') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.publicStreamsToken">
+                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamToken">
                 </label>
                 </label>
-  
+
                 <label class="control-label"><div>${ _('Username') }</div>
                 <label class="control-label"><div>${ _('Username') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.publicStreamsEndpointUrl">
+                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamEndpointUrl">
                 </label>
                 </label>
 
 
                 <label class="control-label"><div>${ _('Object') }</div>
                 <label class="control-label"><div>${ _('Object') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.publicStreamsObject">
+                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamObject">
                 </label>
                 </label>
               </div>
               </div>
-
-              <div class="control-group">
-                <button class="btn" data-bind="click: createWizard.source.publicStreamsCheckConnection">
-                  ${_('Test')}
-                </button>
-              </div>
             <!-- /ko -->
             <!-- /ko -->
+
+            <div class="control-group" data-bind="visible: createWizard.source.hasStreamSelected">
+              <button class="btn" data-bind="click: createWizard.source.streamCheckConnection">
+                ${_('Test')}
+              </button>
+            </div>
           <!-- /ko -->
           <!-- /ko -->
 
 
           <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'table'">
           <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'table'">
@@ -1332,7 +1331,7 @@ ${ assist.assistPanel() }
         self.path('');
         self.path('');
         resizeElements();
         resizeElements();
         self.rdbmsMode('customRdbms');
         self.rdbmsMode('customRdbms');
-        if (val == 'kafka') {
+        if (val == 'stream' && self.streamSelection() == 'kafka') {
           wizard.guessFormat();
           wizard.guessFormat();
           wizard.destination.tableFormat('kudu');
           wizard.destination.tableFormat('kudu');
         }
         }
@@ -1344,8 +1343,7 @@ ${ assist.assistPanel() }
           {'value': 'rdbms', 'name': 'External Database'},
           {'value': 'rdbms', 'name': 'External Database'},
           % endif
           % endif
           % if ENABLE_KAFKA.get():
           % if ENABLE_KAFKA.get():
-          {'value': 'kafka', 'name': 'Internal Stream'},
-          {'value': 'streams', 'name': 'External Stream'},
+          {'value': 'stream', 'name': 'Stream'},
           % endif
           % endif
           % if ENABLE_SQL_INDEXER.get():
           % if ENABLE_SQL_INDEXER.get():
           {'value': 'query', 'name': 'SQL Query'},
           {'value': 'query', 'name': 'SQL Query'},
@@ -1540,7 +1538,13 @@ ${ assist.assistPanel() }
       });
       });
       self.draggedQuery = ko.observable();
       self.draggedQuery = ko.observable();
 
 
-      // Kafka
+      // Streams, Kafka
+      self.publicStreams = ko.observable([
+        {'value': 'kafka', 'name': 'Kafka Topics'},
+        {'value': 'sfdc', 'name': 'SFDC'}
+      ]);
+      self.streamSelection = ko.observable(self.publicStreams()[0]['value']);
+
       self.kafkaTopics = ko.observableArray();
       self.kafkaTopics = ko.observableArray();
       self.kafkaSelectedTopics = ko.observable('');
       self.kafkaSelectedTopics = ko.observable('');
       self.kafkaSelectedTopics.subscribe(function(newValue) {
       self.kafkaSelectedTopics.subscribe(function(newValue) {
@@ -1554,19 +1558,16 @@ ${ assist.assistPanel() }
       self.kafkaFieldTypes = ko.observable('');
       self.kafkaFieldTypes = ko.observable('');
       self.kafkaFieldSchemaPath = ko.observable('');
       self.kafkaFieldSchemaPath = ko.observable('');
 
 
-
-      // Public streams
-      self.publicStreams = ko.observable([
-        'SFDC',
-        'Jira'
-      ]);
-      self.publicStreamsSelection = ko.observable('');
-      self.publicStreamsUsername = ko.observable('');
-      self.publicStreamsPassword = ko.observable('');
-      self.publicStreamsToken = ko.observable('');
-      self.publicStreamsEndpointUrl = ko.observable('');
-      self.publicStreamsObject = ko.observable('');
-      self.publicStreamsCheckConnection = function() {
+      self.streamUsername = ko.observable('');
+      self.streamPassword = ko.observable('');
+      self.streamToken = ko.observable('');
+      self.streamEndpointUrl = ko.observable('');
+      self.streamObject = ko.observable('');
+      self.hasStreamSelected = ko.pureComputed(function() {
+        return (self.streamSelection() == 'kafka' && self.kafkaSelectedTopics()) ||
+           (self.streamSelection() == 'sfdc')
+      });
+      self.streamCheckConnection = function() {
         $.post("${ url('indexer:get_db_component') }", {
         $.post("${ url('indexer:get_db_component') }", {
           "source": ko.mapping.toJSON(self)
           "source": ko.mapping.toJSON(self)
         }, function (resp) {
         }, function (resp) {
@@ -1609,16 +1610,18 @@ ${ assist.assistPanel() }
           return self.query();
           return self.query();
         } else if (self.inputFormat() == 'manual') {
         } else if (self.inputFormat() == 'manual') {
           return true;
           return true;
-        } else if (self.inputFormat() == 'kafka') {
-          return self.kafkaSelectedTopics() && self.kafkaSelectedTopics().length > 0;
-        } else if (self.inputFormat() == 'streams') {
-          return self.publicStreamsSelection() == 'SFDC' ? (
-              self.publicStreamsUsername().length > 0
-              && self.publicStreamsPassword().length > 0
-              && self.publicStreamsToken().length > 0
-              && self.publicStreamsEndpointUrl().length > 0
-              && self.publicStreamsObject().length > 0
-          ) : false;
+        } else if (self.inputFormat() == 'stream') {
+          if (self.streamSelection() == 'kafka') {
+            return self.kafkaSelectedTopics() && self.kafkaSelectedTopics().length > 0;
+          } else {
+            return self.streamSelection() == 'sfdc' &&
+                self.streamUsername().length > 0 &&
+                self.streamPassword().length > 0 &&
+                self.streamToken().length > 0 &&
+                self.streamEndpointUrl().length > 0 &&
+                self.streamObject().length > 0
+              ;
+          }
         } else if (self.inputFormat() == 'rdbms') {
         } else if (self.inputFormat() == 'rdbms') {
           return self.rdbmsDatabaseName().length > 0 && (self.rdbmsTableName().length > 0 || self.rdbmsAllTablesSelected());
           return self.rdbmsDatabaseName().length > 0 && (self.rdbmsTableName().length > 0 || self.rdbmsAllTablesSelected());
         }
         }
@@ -2040,7 +2043,7 @@ ${ assist.assistPanel() }
           } else {
           } else {
             var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
             var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
             self.source.format(newFormat);
             self.source.format(newFormat);
-            if (self.source.inputFormat() == 'kafka') {
+            if (self.source.inputFormat() == 'stream' && self.source.streamSelection() == 'kafka') {
               self.source.kafkaTopics(resp['topics']);
               self.source.kafkaTopics(resp['topics']);
             }
             }
             self.guessFieldTypes();
             self.guessFieldTypes();

Some files were not shown because too many files changed in this diff