Browse Source

HUE-8638 [importer] Add autocomplete of source fields to the editor in the importer

Johan Ahlen 7 years ago
parent
commit
fa4a904f98

+ 18 - 15
desktop/core/src/desktop/static/desktop/js/sqlAutocompleter3.js

@@ -99,6 +99,7 @@ var AutocompleteResults = (function () {
     self.apiHelper = ApiHelper.getInstance();
     self.snippet = options.snippet;
     self.editor = options.editor;
+    self.temporaryOnly = options.snippet.autocompleteSettings && options.snippet.autocompleteSettings.temporaryOnly;
 
     self.sortOverride = null;
 
@@ -268,14 +269,16 @@ var AutocompleteResults = (function () {
     self.handleValues(colRefDeferred);
     self.activeDeferrals.push(self.handlePaths());
 
-    self.activeDeferrals.push(self.handleJoins());
-    self.activeDeferrals.push(self.handleJoinConditions());
-    self.activeDeferrals.push(self.handleAggregateFunctions());
-    self.activeDeferrals.push(self.handleGroupBys(columnsDeferred));
-    self.activeDeferrals.push(self.handleOrderBys(columnsDeferred));
-    self.activeDeferrals.push(self.handleFilters());
-    self.activeDeferrals.push(self.handlePopularTables(tablesDeferred));
-    self.activeDeferrals.push(self.handlePopularColumns(columnsDeferred));
+    if (!self.temporaryOnly) {
+      self.activeDeferrals.push(self.handleJoins());
+      self.activeDeferrals.push(self.handleJoinConditions());
+      self.activeDeferrals.push(self.handleAggregateFunctions());
+      self.activeDeferrals.push(self.handleGroupBys(columnsDeferred));
+      self.activeDeferrals.push(self.handleOrderBys(columnsDeferred));
+      self.activeDeferrals.push(self.handleFilters());
+      self.activeDeferrals.push(self.handlePopularTables(tablesDeferred));
+      self.activeDeferrals.push(self.handlePopularColumns(columnsDeferred));
+    }
 
     $.when.apply($, self.activeDeferrals).always(function () {
       huePubSub.publish('hue.ace.autocompleter.done');
@@ -326,7 +329,7 @@ var AutocompleteResults = (function () {
   AutocompleteResults.prototype.loadDatabases = function () {
     var self = this;
     var databasesDeferred = $.Deferred();
-    DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [] }).done(function (entry) {
+    DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [], temporaryOnly: self.temporaryOnly }).done(function (entry) {
       self.cancellablePromises.push(entry.getChildren({ silenceErrors: true, cancellable: true }).done(function (databases) {
         databasesDeferred.resolve(databases);
       }).fail(databasesDeferred.reject));
@@ -544,7 +547,7 @@ var AutocompleteResults = (function () {
 
         var database = suggestTables.identifierChain && suggestTables.identifierChain.length === 1 ? suggestTables.identifierChain[0].name : self.activeDatabase;
 
-        DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [ database ]}).done(function (dbEntry) {
+        DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [ database ], temporaryOnly: self.temporaryOnly }).done(function (dbEntry) {
           self.cancellablePromises.push(dbEntry.getChildren({ silenceErrors: true, cancellable: true }).done(function (tableEntries) {
             var tableSuggestions = [];
 
@@ -1364,7 +1367,7 @@ var AutocompleteResults = (function () {
         && self.parseResult.suggestTables.identifierChain.length === 1
         && self.parseResult.suggestTables.identifierChain[0].name ? self.parseResult.suggestTables.identifierChain[0].name : self.activeDatabase;
 
-      DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [ db ]}).done(function (entry) {
+      DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [ db ], temporaryOnly: self.temporaryOnly }).done(function (entry) {
         self.cancellablePromises.push(entry.loadNavOptPopularityForChildren({ silenceErrors: true, cancellable: true }).done(function (childEntries) {
           var totalPopularity = 0;
           var popularityIndex = {};
@@ -1587,7 +1590,7 @@ var AutocompleteResults = (function () {
         }
       }
 
-      DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: fetchedPath }).done(function (catalogEntry) {
+      DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: fetchedPath, temporaryOnly: self.temporaryOnly }).done(function (catalogEntry) {
         self.cancellablePromises.push(catalogEntry.getSourceMeta({ silenceErrors: true, cancellable: true }).done(function (sourceMeta) {
           if (self.snippet.type() === 'hive'
               && typeof sourceMeta.extended_columns !== 'undefined'
@@ -1617,7 +1620,7 @@ var AutocompleteResults = (function () {
     // For Hive it could be either:
     // SELECT col.struct FROM db.tbl -or- SELECT col.struct FROM tbl
     if (path.length > 1 && (self.snippet.type() === 'impala' || self.snippet.type() === 'hive')) {
-      DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [] }).done(function (catalogEntry) {
+      DataCatalog.getEntry({ sourceType: self.snippet.type(), namespace: self.snippet.namespace(), compute: self.snippet.compute(), path: [], temporaryOnly: self.temporaryOnly }).done(function (catalogEntry) {
         self.cancellablePromises.push(catalogEntry.getChildren({ silenceErrors: true, cancellable: true }).done(function (databaseEntries) {
           var firstIsDb = databaseEntries.some(function (dbEntry) {
             return hueUtils.equalIgnoreCase(dbEntry.name, path[0]);
@@ -1645,8 +1648,8 @@ var SqlAutocompleter3 = (function () {
   /**
    * @param {Object} options
    * @param {Snippet} options.snippet
-   * @param {string) [options.fixedPrefix] - Optional prefix to always use on parse
-   * @param {string) [options.fixedPostfix] - Optional postfix to always use on parse
+   * @param {string} [options.fixedPrefix] - Optional prefix to always use on parse
+   * @param {string} [options.fixedPostfix] - Optional postfix to always use on parse
    * @constructor
    */
   function SqlAutocompleter3(options) {

+ 6 - 0
desktop/core/src/desktop/static/desktop/spec/sqlAutocompleter3Spec.js

@@ -20,6 +20,9 @@
 
       var subject = new AutocompleteResults({
         snippet: {
+          autocompleteSettings: {
+            temporaryOnly: false
+          },
           type: function () {
             return 'hive';
           },
@@ -413,6 +416,9 @@
 
         return new SqlAutocompleter3({
           snippet: {
+            autocompleteSettings: {
+              temporaryOnly: false
+            }
             type: function () {
               return dialect;
             },

+ 7 - 2
desktop/core/src/desktop/templates/ko_components/ko_simple_ace_editor.mako

@@ -775,7 +775,9 @@ from desktop.views import _ko
         'solrFormula': SolrFormulaAutocompleter,
         'solrQuery':  SolrQueryAutocompleter,
         'impalaQuery': SqlAutocompleter3,
-        'hiveQuery': SqlAutocompleter3
+        'hiveQuery': SqlAutocompleter3,
+        'impala': SqlAutocompleter3,
+        'hive': SqlAutocompleter3
       };
 
       var SimpleAceEditor = function (params, element) {
@@ -807,11 +809,14 @@ from desktop.views import _ko
             throw new Error('Could not find autocompleter for "' + params.autocomplete.type + '"');
           }
 
-          var sourceType = params.autocomplete.type.substring(0, params.autocomplete.type.indexOf('Query'));
+          var sourceType = params.autocomplete.type.indexOf('Query') !== -1 ? params.autocomplete.type.replace('Query', '') : params.autocomplete.type;
 
           var autocompleteArgs = {
             editor: function() { return editor },
             snippet: {
+              autocompleteSettings: {
+                temporaryOnly: params.temporaryOnly
+              },
               type: function () {
                 return sourceType;
               },

+ 45 - 3
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -981,8 +981,8 @@ ${ assist.assistPanel() }
 
               <div data-bind="component: { name: 'hue-simple-ace-editor-multi', params: {
                   value: fieldEditorValue,
-                  placeHolder: '${ _ko('Example: SELECT a, b FROM c') }',
-                  autocomplete: { type: 'hiveQuery' },
+                  placeHolder: '${ _ko('Example: SELECT field_1 FROM input') }',
+                  autocomplete: { type: sourceType },
                   lines: 5,
                   aceOptions: {
                     minLines: 10,
@@ -991,7 +991,8 @@ ${ assist.assistPanel() }
                   database: fieldEditorDatabase,
                   namespace: namespace,
                   compute: compute,
-                  mode: 'hive'
+                  temporaryOnly: true,
+                  mode: sourceType
                 }}"></div>
               <!-- /ko -->
               <!-- ko ifnot: useFieldEditor -->
@@ -1610,6 +1611,47 @@ ${ assist.assistPanel() }
       self.namespace = wizard.namespace;
       self.compute = wizard.compute;
 
+      var refreshThrottle = -1;
+      var sampleColSubDisposals = [];
+      var refreshTemporaryTable = function (sampleCols) {
+        window.clearTimeout(refreshThrottle);
+        window.setTimeout(function () {
+          while (sampleColSubDisposals.length) {
+            sampleColSubDisposals.pop()();
+          }
+          var temporaryColumns = [];
+          sampleCols.forEach(function (sampleCol) {
+            var col = {
+              name: sampleCol.name(),
+              type: sampleCol.type()
+            };
+            temporaryColumns.push(col);
+            var colNameSub = sampleCol.name.subscribe(function () {
+              refreshTemporaryTable(self.sampleCols())
+            });
+            var colTypeSub = sampleCol.type.subscribe(function () {
+              refreshTemporaryTable(self.sampleCols())
+            });
+            sampleColSubDisposals.push(function () {
+              colNameSub.dispose();
+              colTypeSub.dispose();
+            })
+          });
+          var handle = DataCatalog.addTemporaryTable({
+            sourceType: self.sourceType,
+            namespace: self.namespace(),
+            compute: self.compute(),
+            name: 'input',
+            columns: temporaryColumns
+          });
+          sampleColSubDisposals.push(function () {
+            handle.delete();
+          })
+        }, 500)
+      };
+
+
+      self.sampleCols.subscribe(refreshTemporaryTable);
       self.inputFormat = ko.observable(wizard.prefill.source_type() ? wizard.prefill.source_type() : 'file');
 
       self.inputFormat.subscribe(function(val) {

+ 4 - 0
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -325,6 +325,10 @@ var EditorViewModel = (function() {
           || $.grep(vm.availableLanguages, function(language) { return language.type == self.type() && language.interface == 'oozie'; }).length > 0;
     });
 
+    self.autocompleteSettings = {
+      temporaryOnly: false
+    };
+
     // Ace stuff
     self.aceCursorPosition = ko.observable(notebook.isHistory() ? snippet.aceCursorPosition : null);