Browse Source

HUE-4918 [importer] Invalidate Impala metadata on new table creation in Hive and Impala

This will send invalidate for the database used when creating a new table (Hive or Impala) from the importer.
Johan Ahlen 8 years ago
parent
commit
10c1219

+ 13 - 8
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -99,7 +99,11 @@ var ApiHelper = (function () {
     self.user = user;
     self.user = user;
     self.lastKnownDatabases = {};
     self.lastKnownDatabases = {};
     self.queueManager = ApiQueueManager.getInstance();
     self.queueManager = ApiQueueManager.getInstance();
-    self.invalidateImpala = 'cache';
+    self.invalidateImpala = null;
+
+    huePubSub.subscribe('assist.invalidate.impala', function (details) {
+      self.invalidateImpala = details;
+    });
 
 
     huePubSub.subscribe('assist.clear.db.cache', function (options) {
     huePubSub.subscribe('assist.clear.db.cache', function (options) {
       self.clearDbCache(options);
       self.clearDbCache(options);
@@ -891,7 +895,6 @@ var ApiHelper = (function () {
    * @param {Object} options
    * @param {Object} options
    * @param {string} options.sourceType
    * @param {string} options.sourceType
    * @param {string} [options.databaseName]
    * @param {string} [options.databaseName]
-   * @param {boolean} [options.invalidateImpala]
    * @param {string} [options.tableName]
    * @param {string} [options.tableName]
    * @param {string} [options.cacheType] - Possible values 'default', 'optimizer. Default value 'default'
    * @param {string} [options.cacheType] - Possible values 'default', 'optimizer. Default value 'default'
    * @param {string[]} [options.fields]
    * @param {string[]} [options.fields]
@@ -899,7 +902,6 @@ var ApiHelper = (function () {
    */
    */
   ApiHelper.prototype.clearDbCache = function (options) {
   ApiHelper.prototype.clearDbCache = function (options) {
     var self = this;
     var self = this;
-    self.invalidateImpala = options.invalidateImpala || 'cache';
     var cacheIdentifier = self.getAssistCacheIdentifier(options);
     var cacheIdentifier = self.getAssistCacheIdentifier(options);
     if (options.clearAll) {
     if (options.clearAll) {
       $.totalStorage(cacheIdentifier, {});
       $.totalStorage(cacheIdentifier, {});
@@ -933,7 +935,7 @@ var ApiHelper = (function () {
     var self = this;
     var self = this;
 
 
     var loadFunction = function () {
     var loadFunction = function () {
-      self.invalidateImpala = 'cache';
+
       fetchAssistData.bind(self)($.extend({}, options, {
       fetchAssistData.bind(self)($.extend({}, options, {
         url: AUTOCOMPLETE_API_PREFIX,
         url: AUTOCOMPLETE_API_PREFIX,
         successCallback: function (data) {
         successCallback: function (data) {
@@ -960,10 +962,13 @@ var ApiHelper = (function () {
       }));
       }));
     };
     };
 
 
-    if (options.sourceType === 'impala' && self.invalidateImpala == 'invalidateAndFlush') {
-      $.post(IMPALA_INVALIDATE_API, { flush_all: true, database: options.database }, loadFunction);
-    } else if (options.sourceType === 'impala' && self.invalidateImpala == 'invalidate') {
-      $.post(IMPALA_INVALIDATE_API, { flush_all: false, database: options.database }, loadFunction);
+    if (options.sourceType === 'impala' && self.invalidateImpala !== null) {
+      if (self.invalidateImpala.database) {
+        $.post(IMPALA_INVALIDATE_API, { flush_all:  self.invalidateImpala.flush, database: self.invalidateImpala.database }, loadFunction);
+      } else {
+        $.post(IMPALA_INVALIDATE_API, { flush_all:  self.invalidateImpala.flush }, loadFunction);
+      }
+      self.invalidateImpala = null;
     } else {
     } else {
       loadFunction();
       loadFunction();
     }
     }

+ 8 - 2
desktop/core/src/desktop/static/desktop/js/assist/assistDbSource.js

@@ -280,11 +280,17 @@ var AssistDbSource = (function () {
     };
     };
 
 
     self.reload = function(allCacheTypes) {
     self.reload = function(allCacheTypes) {
+      if (self.invalidateOnRefresh() !== 'cache') {
+        huePubSub.publish('assist.invalidate.impala', {
+          flush: self.invalidateOnRefresh() === 'invalidateAndFlush',
+          database: self.selectedDatabase() ? self.selectedDatabase().definition.name : null
+        });
+      }
+
       self.reloading(true);
       self.reloading(true);
       huePubSub.publish('assist.clear.db.cache', {
       huePubSub.publish('assist.clear.db.cache', {
         sourceType: self.sourceType,
         sourceType: self.sourceType,
-        clearAll: true,
-        invalidateImpala: self.invalidateOnRefresh()
+        clearAll: true
       });
       });
       if (allCacheTypes) {
       if (allCacheTypes) {
         huePubSub.publish('assist.clear.db.cache', {
         huePubSub.publish('assist.clear.db.cache', {

+ 13 - 6
desktop/core/src/desktop/templates/ko_components.mako

@@ -334,6 +334,15 @@ from desktop.views import _ko
                 //self.indexingSuccess(true);
                 //self.indexingSuccess(true);
               }
               }
             });
             });
+
+            var refreshAssist = function (snippet) {
+              var match = snippet.statement_raw().match(/CREATE TABLE `([^`]+)`/i);
+              if (match) {
+                var db = match[1];
+                huePubSub.publish('assist.invalidate.impala', { flush: false, database: db });
+              }
+              huePubSub.publish('assist.db.refresh', { sourceTypes: ['hive', 'impala'] });
+            };
             notebook.snippets()[0].status.subscribe(function(val){
             notebook.snippets()[0].status.subscribe(function(val){
               if (val == 'failed'){
               if (val == 'failed'){
                 //self.isIndexing(false);
                 //self.isIndexing(false);
@@ -344,15 +353,13 @@ from desktop.views import _ko
                 if (! snippet.result.handle().has_more_statements) {
                 if (! snippet.result.handle().has_more_statements) {
                   // TODO: Show finish notification and clicking on it does onSuccessUrl
                   // TODO: Show finish notification and clicking on it does onSuccessUrl
                   // or if still on initial spinner we redirect automatically to onSuccessUrl
                   // or if still on initial spinner we redirect automatically to onSuccessUrl
-                  if (notebook.onSuccessUrl()) {
-                    if (notebook.onSuccessUrl() == 'assist.db.refresh') { // TODO: Similar if in in FB directory, also refresh FB dir
-                      huePubSub.publish('assist.db.refresh', { sourceTypes: ['hive', 'impala'] });
-                    } else {
-                      huePubSub.publish('open.link', notebook.onSuccessUrl());
-                    }
+                  refreshAssist(snippet);
+                  if (notebook.onSuccessUrl() && notebook.onSuccessUrl() !== 'assist.db.refresh') { // TODO: Similar if in in FB directory, also refresh FB dir
+                    huePubSub.publish('open.link', notebook.onSuccessUrl());
                   }
                   }
                 } else { // Perform last DROP statement execute
                 } else { // Perform last DROP statement execute
                   snippet.execute();
                   snippet.execute();
+                  refreshAssist(snippet);
                 }
                 }
               }
               }
             });
             });

+ 14 - 9
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -1389,7 +1389,7 @@ ${ assist.assistPanel() }
         catch (err) {
         catch (err) {
         }
         }
         $('#fieldsBulkEditor').modal('hide');
         $('#fieldsBulkEditor').modal('hide');
-      }
+      };
 
 
       self.isTargetExisting = ko.observable();
       self.isTargetExisting = ko.observable();
       self.isTargetChecking = ko.observable(false);
       self.isTargetChecking = ko.observable(false);
@@ -1681,7 +1681,12 @@ ${ assist.assistPanel() }
                   var snippet = self.editorVM.selectedNotebook().snippets()[0]; // Could be native to editor at some point
                   var snippet = self.editorVM.selectedNotebook().snippets()[0]; // Could be native to editor at some point
                   if (! snippet.result.handle().has_more_statements) {
                   if (! snippet.result.handle().has_more_statements) {
                     if (self.editorVM.selectedNotebook().onSuccessUrl()) {
                     if (self.editorVM.selectedNotebook().onSuccessUrl()) {
-                      huePubSub.publish('assist.clear.db.cache', {sourceType: self.source.apiHelperType()});
+                      var match = snippet.statement_raw().match(/CREATE TABLE `([^`]+)`/i);
+                      if (match) {
+                        var db = match[1];
+                        huePubSub.publish('assist.invalidate.impala', { flush: false, database: db });
+                      }
+                      huePubSub.publish('assist.clear.db.cache', { sourceType: self.source.apiHelperType() });
                       window.location.href = self.editorVM.selectedNotebook().onSuccessUrl();
                       window.location.href = self.editorVM.selectedNotebook().onSuccessUrl();
                     }
                     }
                   } else { // Perform last DROP statement execute
                   } else { // Perform last DROP statement execute
@@ -1716,17 +1721,17 @@ ${ assist.assistPanel() }
 % endif
 % endif
 
 
         hueAnalytics.log('importer', 'submit/' + self.source.inputFormat() + '/' + self.destination.outputFormat());
         hueAnalytics.log('importer', 'submit/' + self.source.inputFormat() + '/' + self.destination.outputFormat());
-      }
+      };
 
 
       self.removeOperation = function (operation, operationList) {
       self.removeOperation = function (operation, operationList) {
         operationList.remove(operation);
         operationList.remove(operation);
         hueAnalytics.log('importer', 'step/removeOperation');
         hueAnalytics.log('importer', 'step/removeOperation');
-      }
+      };
 
 
       self.addOperation = function (field) {
       self.addOperation = function (field) {
         field.operations.push(new Operation("split"));
         field.operations.push(new Operation("split"));
         hueAnalytics.log('importer', 'step/addOperation');
         hueAnalytics.log('importer', 'step/addOperation');
-      }
+      };
 
 
       self.load = function (state) {
       self.load = function (state) {
         self.source.name(state.name);
         self.source.name(state.name);
@@ -1750,7 +1755,7 @@ ${ assist.assistPanel() }
         options.name = getNewFieldName();
         options.name = getNewFieldName();
       }
       }
       return loadField($.extend({}, ${default_field_type | n}, options));
       return loadField($.extend({}, ${default_field_type | n}, options));
-    }
+    };
 
 
     var loadField = function (currField, parent, idx) {
     var loadField = function (currField, parent, idx) {
       var koField = ko.mapping.fromJS(currField);
       var koField = ko.mapping.fromJS(currField);
@@ -1788,7 +1793,7 @@ ${ assist.assistPanel() }
       koField.keyType.subscribe(autoExpand);
       koField.keyType.subscribe(autoExpand);
 
 
       return koField;
       return koField;
-    }
+    };
 
 
     var IndexerViewModel = function () {
     var IndexerViewModel = function () {
       var self = this;
       var self = this;
@@ -1818,13 +1823,13 @@ ${ assist.assistPanel() }
           self.currentStep(self.currentStep() + 1);
           self.currentStep(self.currentStep() + 1);
           hueAnalytics.log('importer', 'step/' + self.currentStep());
           hueAnalytics.log('importer', 'step/' + self.currentStep());
         }
         }
-      }
+      };
       self.previousStep = function () {
       self.previousStep = function () {
         if (self.previousStepVisible()){
         if (self.previousStepVisible()){
           self.currentStep(self.currentStep() - 1);
           self.currentStep(self.currentStep() - 1);
           hueAnalytics.log('importer', 'step/' + self.currentStep());
           hueAnalytics.log('importer', 'step/' + self.currentStep());
         }
         }
-      }
+      };
 
 
       self.isLoading = ko.observable(false);
       self.isLoading = ko.observable(false);