Browse Source

HUE-7820 [frontend] Add navopt metadata to the datacatalog and make all promises cancellable

Johan Ahlen 7 years ago
parent
commit
a1f8e0039b

+ 167 - 47
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -49,6 +49,70 @@ var ApiQueueManager = (function () {
   };
 })();
 
+var CancellablePromise = (function () {
+
+  function CancellablePromise(promise, request, otherCancellables) {
+    var self = this;
+    self.promise = promise;
+    self.request = request;
+    self.otherCancellables = otherCancellables;
+  }
+
+  CancellablePromise.prototype.cancel = function () {
+    var self = this;
+    if (self.request) {
+      ApiHelper.getInstance().cancelActiveRequest(self.request);
+    }
+    if (self.otherCancellables) {
+      self.otherCancellables.forEach(function (cancellable) { cancellable.cancel() });
+    }
+    return this;
+  };
+
+  CancellablePromise.prototype.then = function () {
+    var self = this;
+    self.promise.then.apply(self.promise, arguments);
+    return self;
+  };
+
+  CancellablePromise.prototype.done = function (callback) {
+    var self = this;
+    self.promise.done.apply(self.promise, arguments);
+    return self;
+  };
+
+  CancellablePromise.prototype.fail = function (callback) {
+    var self = this;
+    self.promise.fail.apply(self.promise, arguments);
+    return self;
+  };
+
+  CancellablePromise.prototype.always = function (callback) {
+    var self = this;
+    self.promise.always.apply(self.promise, arguments);
+    return self;
+  };
+
+  CancellablePromise.prototype.pipe = function (callback) {
+    var self = this;
+    self.promise.pipe.apply(self.promise, arguments);
+    return self;
+  };
+
+  CancellablePromise.prototype.progress = function (callback) {
+    var self = this;
+    self.promise.progress.apply(self.promise, arguments);
+    return self;
+  };
+
+  CancellablePromise.prototype.state = function () {
+    var self = this;
+    return self.promise.state();
+  };
+
+  return CancellablePromise;
+})();
+
 var ApiHelper = (function () {
 
   var AUTOCOMPLETE_API_PREFIX = "/notebook/api/autocomplete/";
@@ -1473,27 +1537,27 @@ var ApiHelper = (function () {
    *
    * @param {string[]} [options.path] - The path to fetch
    *
-   * @return {Deferred} Promise
+   * @return {CancellablePromise}
    */
   ApiHelper.prototype.fetchSourceMetadata = function (options) {
     var self = this;
-    var promise = $.Deferred();
+    var deferred = $.Deferred();
 
-    fetchAssistData.bind(self)({
+    var request = fetchAssistData.bind(self)({
       url: AUTOCOMPLETE_API_PREFIX + options.path.join('/'),
       sourceType: options.sourceType,
       silenceErrors: options.silenceErrors,
       cachedOnly: options.cachedOnly,
       refreshCache: options.refreshCache,
-      successCallback: promise.resolve,
+      successCallback: deferred.resolve,
       errorCallback: self.assistErrorCallback({
-        errorCallback: promise.reject,
+        errorCallback: deferred.reject,
         silenceErrors: options.silenceErrors
       }),
       cacheCondition: genericCacheCondition
     });
 
-    return promise;
+    return new CancellablePromise(deferred.promise(), request);
   };
 
 
@@ -1547,11 +1611,13 @@ var ApiHelper = (function () {
    *
    * @param {string} options.sourceType
    * @param {string[]} options.path
+   *
+   * @return {CancellablePromise}
    */
   ApiHelper.prototype.fetchSample = function (options) {
     var self = this;
     var deferred = $.Deferred();
-    fetchAssistData.bind(self)({
+    var request = fetchAssistData.bind(self)({
       url: SAMPLE_API_PREFIX + options.path.join('/'),
       sourceType: options.sourceType,
       noCache: options.noCache,
@@ -1564,7 +1630,7 @@ var ApiHelper = (function () {
       }),
       cacheCondition: genericCacheCondition
     });
-    return deferred.promise();
+    return new CancellablePromise(deferred.promise(), request);
   };
 
   /**
@@ -1577,6 +1643,8 @@ var ApiHelper = (function () {
    *
    * @param {boolean} [options.isView] - Default false
    * @param {string[]} options.path
+   *
+   * @return {CancellablePromise}
    */
   ApiHelper.prototype.fetchNavigatorMetadata = function (options) {
     var self = this;
@@ -1591,7 +1659,7 @@ var ApiHelper = (function () {
       url +=  '?type=field&database=' + options.path[0] + '&table=' + options.path[1] + '&name=' + options.path[2];
     }
 
-    fetchAssistData.bind(self)({
+    var request = fetchAssistData.bind(self)({
       url: url,
       sourceType: 'nav',
       noCache: options.noCache,
@@ -1607,7 +1675,7 @@ var ApiHelper = (function () {
       cacheCondition: genericCacheCondition
     });
 
-    return deferred.promise();
+    return new CancellablePromise(deferred.promise(), request);
   };
 
   ApiHelper.prototype.updateNavigatorMetadata = function (options) {
@@ -1633,6 +1701,32 @@ var ApiHelper = (function () {
     });
   };
 
+  /**
+   * Fetches navOpt metadata for the children of the given path
+   *
+   * @param {Object} options
+   * @param {boolean} [options.silenceErrors]
+   * @param {boolean} [options.refreshCache] - Default false
+   * @param {string[]} options.path
+   * @return {CancellablePromise}
+   */
+  ApiHelper.prototype.fetchNavOptMetadata = function (options) {
+    var self = this;
+    var deferred = $.Deferred();
+    var url = options.path.length === 1 ? NAV_OPT_URLS.TOP_TABLES : NAV_OPT_URLS.TOP_COLUMNS;
+    return self.fetchNavOptCached(url, {
+      path: options.path,
+      refreshCache: options.refreshCache,
+      silenceErrors: options.silenceErrors,
+      successCallback: deferred.resolve,
+      errorCallback: deferred.reject
+    }, function (data) {
+      return data.status === 0;
+    });
+  };
+
+
+
   /**
    * Lists all available navigator tags
    *
@@ -1985,21 +2079,19 @@ var ApiHelper = (function () {
     var self = this;
 
     var performFetch = function (data, hash) {
-      var promise = self.queueManager.getQueued(url, hash);
-      var firstInQueue = typeof promise === 'undefined';
-      if (firstInQueue) {
-        promise = $.Deferred();
-        self.queueManager.addToQueue(promise, url, hash);
-      }
+      var queuedPromise = self.queueManager.getQueued(url, hash);
 
-      promise.done(options.successCallback).fail(self.assistErrorCallback(options)).always(function () {
-        if (typeof options.editor !== 'undefined' && options.editor !== null) {
-          options.editor.hideSpinner();
+      if (queuedPromise) {
+        if (options.successCallback) {
+          queuedPromise.done(options.successCallback);
         }
-      });
-
-      if (!firstInQueue) {
-        return;
+        queuedPromise.fail(self.assistErrorCallback(options));
+        if (options.editor) {
+          queuedPromise.always(function () {
+            options.editor.hideSpinner();
+          })
+        }
+        return queuedPromise;
       }
 
       var fetchFunction = function (storeInCache) {
@@ -2012,43 +2104,70 @@ var ApiHelper = (function () {
           type: 'post',
           url: url,
           data: data,
-          timeout: options.timeout
-        })
-          .done(function (data) {
-            if (data.status === 0) {
-              if (cacheCondition(data)) {
-                storeInCache(data);
-              }
-              promise.resolve(data);
-            } else {
-              promise.reject(data);
+          timeout: options.timeout || AUTOCOMPLETE_TIMEOUT
+        }).done(function (data) {
+          if (data.status === 0) {
+            if (cacheCondition(data)) {
+              storeInCache(data);
             }
-          })
-          .fail(promise.reject);
+            promise.resolve(data);
+          } else {
+            promise.reject(data);
+          }
+        })
+        .fail(promise.reject);
       };
 
-      return fetchCached.bind(self)($.extend({}, options, {
+      var deferred = $.Deferred();
+
+      var request = fetchCached.bind(self)($.extend({}, options, {
         url: url,
         hash: hash,
         cacheType: 'optimizer',
         fetchFunction: fetchFunction,
-        promise: promise
+        promise: deferred
       }));
-    }
 
-    var promise = $.Deferred();
-    if (options.tables) {
+      var promise = new CancellablePromise(deferred.promise(), request);
+
+      self.queueManager.addToQueue(promise, url, hash);
+      return promise;
+    };
+
+    var deferred = $.Deferred();
+    var request;
+
+    if (options.tables && options.tables.identifierChain) {
       self.createNavOptDbTablesJson(options).done(function (json) {
-        promise.resolve(performFetch({
+        deferred.resolve(performFetch({
           dbTables: json
         }, json.hashCode()))
       });
     } else if (options.database) {
-      promise.resolve(performFetch({
+      request = performFetch({
         database: options.database
-      }, options.database));
+      }, options.database).done(deferred.resolve);
+      deferred.resolve();
+    } else if (options.path) {
+      var data, hash;
+      if (options.path.length === 1) {
+        data = {
+          database: options.path[0]
+        };
+        hash = data.database;
+      } else if (options.path.length === 2) {
+        data = {
+          dbTables: ko.mapping.toJSON([options.path.join('.')])
+        };
+        hash = data.dbTables.hashCode();
+      } else {
+        deferred.reject();
+        return deferred.promise();
+      }
+      request = performFetch(data, hash).done(deferred.resolve);
     }
-    return promise;
+
+    return new CancellablePromise(deferred.promise(), request);
   };
 
   ApiHelper.prototype.fetchHueDocsInteractive = function (query) {
@@ -2087,7 +2206,7 @@ var ApiHelper = (function () {
     var self = this;
     var deferred = $.Deferred();
 
-    $.post(SEARCH_API, {
+    var request = $.post(SEARCH_API, {
       query_s: ko.mapping.toJSON(options.query),
       limit: options.limit || 100,
       raw_query: !!options.rawQuery,
@@ -2105,7 +2224,7 @@ var ApiHelper = (function () {
       silenceErrors: options.silenceErrors,
       errorCallback: deferred.reject
     }));
-    return deferred.promise();
+    return new CancellablePromise(deferred.promise(), request);
   };
 
   ApiHelper.prototype.formatSql = function (statements) {
@@ -2220,6 +2339,7 @@ var ApiHelper = (function () {
    * @param {Object} options
    * @param {string} options.sourceType
    * @param {string} options.url
+   * @param {boolean} options.refreshCache
    * @param {string} [options.hash] - Optional hash to use as well as the url
    * @param {Function} options.fetchFunction
    * @param {Function} options.successCallback
@@ -2233,7 +2353,7 @@ var ApiHelper = (function () {
     var cachedData = $.totalStorage(cacheIdentifier) || {};
     var cachedId = options.hash ? options.url + options.hash : options.url;
 
-    if (typeof cachedData[cachedId] == "undefined" || self.hasExpired(cachedData[cachedId].timestamp, options.cacheType || 'default')) {
+    if (options.refreshCache || typeof cachedData[cachedId] == "undefined" || self.hasExpired(cachedData[cachedId].timestamp, options.cacheType || 'default')) {
       if (typeof options.editor !== 'undefined' && options.editor !== null) {
         options.editor.showSpinner();
       }

+ 97 - 35
desktop/core/src/desktop/static/desktop/js/dataCatalog.js

@@ -55,7 +55,8 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   *
+   * @return {CancellablePromise}
    */
   var reloadSourceMeta = function (dataCatalogEntry, apiOptions) {
     dataCatalogEntry.lastSourceMetaPromise = fetchMeta('fetchSourceMetadata', dataCatalogEntry, apiOptions);
@@ -71,7 +72,8 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   *
+   * @return {CancellablePromise}
    */
   var reloadNavigatorMeta = function (dataCatalogEntry, apiOptions) {
     if (HAS_NAVIGATOR) {
@@ -91,7 +93,8 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   *
+   * @return {CancellablePromise}
    */
   var reloadSample = function (dataCatalogEntry, apiOptions) {
     dataCatalogEntry.lastSamplePromise = fetchMeta('fetchSample', dataCatalogEntry, apiOptions);
@@ -121,10 +124,14 @@ var DataCatalog = (function () {
 
     self.lastNavigatorMetaPromise = undefined;
     self.navigatorMeta = undefined;
+    self.navigatorMetaForChildrenPromise = undefined;
 
     self.lastSamplePromise = undefined;
     self.sample = undefined;
 
+    self.navOptMeta = undefined;
+    self.navOptMetaForChildrenPromise = undefined;
+
     self.children = undefined;
   }
 
@@ -133,11 +140,13 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   *
+   * @return {CancellablePromise}
    */
   DataCatalogEntry.prototype.getChildren = function (apiOptions) {
     var self = this;
     var deferred = $.Deferred();
+    var cancellablePromises = [];
     if (self.children) {
       deferred.resolve(self.children);
     } else {
@@ -150,18 +159,20 @@ var DataCatalog = (function () {
           var entities = sourceMeta.databases || sourceMeta.tables_meta || sourceMeta.extended_columns || sourceMeta.fields;
           if (entities) {
             entities.forEach(function (entity) {
+              var catalogEntry = self.dataCatalog.getEntry({ path: self.path.concat(entity.name || entity) });
               var definition = typeof entity === 'object' ? entity : {};
               definition.index = index++;
-              var catalogEntry = self.dataCatalog.getEntry({ path: self.path.concat(entity.name || entity), definition: definition });
+              catalogEntry.definition = definition; // Override any existing definition as parent might know more
               self.children.push(catalogEntry);
             });
           } else {
             (sourceMeta.type === 'map' ? ['key', 'value'] : ['item']).forEach(function (path) {
               if (sourceMeta[path]) {
+                var catalogEntry = self.dataCatalog.getEntry({ path: self.path.concat(path) });
                 var definition = sourceMeta[path];
                 definition.index = index++;
-                definition.isMapValue = path === 'value'
-                var catalogEntry = self.dataCatalog.getEntry({ path: self.path.concat(path), definition: definition.isMapValue = path === 'value' });
+                definition.isMapValue = path === 'value';
+                catalogEntry.definition = definition; // Override any existing definition as parent might know more
                 self.children.push(catalogEntry);
               }
             })
@@ -175,15 +186,24 @@ var DataCatalog = (function () {
 
   /**
    * @param {Object} [apiOptions]
+   * @param {boolean} [apiOptions.refreshCache]
    * @param {boolean} [apiOptions.silenceErrors]
-   * @return {Promise}
+   *
+   * @return {CancellablePromise}
    */
   DataCatalogEntry.prototype.loadNavigatorMetaForChildren = function (apiOptions) {
     var self = this;
+
+    if (self.navigatorMetaForChildrenPromise && (!apiOptions || !apiOptions.refreshCache)) {
+      return self.navigatorMetaForChildrenPromise;
+    }
+
     var deferred = $.Deferred();
-    self.getChildren().done(function (children) {
-      var query;
 
+    var cancellablePromises = [];
+
+    cancellablePromises.push(self.getChildren(apiOptions).done(function (children) {
+      var query;
       // TODO: Add sourceType to nav search query
       if (self.path.length) {
         query = 'parentPath:"/' + self.path.join('/') + '" AND type:(table view field)';
@@ -191,29 +211,71 @@ var DataCatalog = (function () {
         query = 'type:database'
       }
 
-      ApiHelper.getInstance().searchEntities({
+      cancellablePromises.push(ApiHelper.getInstance().searchEntities({
         query: query,
         rawQuery: true,
         limit: children.length,
         silenceErrors: apiOptions && apiOptions.silenceErrors
       }).done(function (result) {
+        var entriesWithNavMeta = [];
         if (result && result.entities && result.entities.length > 0) {
-          var entityIndex = {};
           result.entities.forEach(function (entity) {
-            entityIndex[(entity.name || entity.originalName).toLowerCase()] = entity;
+            var entry = self.dataCatalog.getEntry({ path: self.path.concat((entity.name || entity.originalName).toLowerCase())});
+            entry.navigatorMeta = entity;
+            entry.lastNavigatorMetaPromise = $.Deferred().resolve(entry.navigatorMeta).promise();
+            entriesWithNavMeta.push(entry);
           });
-          children.forEach(function (child) {
-            var name = child.name.toLowerCase();
-            if (entityIndex[name]) {
-              child.navigatorMeta = entityIndex[name];
-              child.lastNavigatorMetaPromise = $.Deferred().resolve(child.navigatorMeta).promise();
-            }
+        }
+        deferred.resolve(entriesWithNavMeta);
+      }).fail(deferred.reject));
+    }).fail(deferred.reject));
+
+
+    self.navigatorMetaForChildrenPromise = new CancellablePromise(deferred.promise(), null, cancellablePromises);
+    return self.navigatorMetaForChildrenPromise;
+  };
+
+  /**
+   * @param {Object} [apiOptions]
+   * @param {boolean} [apiOptions.refreshCache]
+   * @param {boolean} [apiOptions.silenceErrors]
+   *
+   * @return {CancellablePromise}
+   */
+  DataCatalogEntry.prototype.loadNavOptMetaForChildren = function (apiOptions) {
+    var self = this;
+    if (self.navOptMetaForChildrenPromise && (!apiOptions || !apiOptions.refreshCache)) {
+      return self.navOptMetaForChildrenPromise;
+    }
+    var deferred = $.Deferred();
+    var cancellablePromises = [];
+    if (self.isDatabase() || self.isTableOrView()) {
+      cancellablePromises.push(ApiHelper.getInstance().fetchNavOptMetadata({
+        silenceErrors: apiOptions && apiOptions.silenceErrors,
+        refreshCache: apiOptions && apiOptions.refreshCache,
+        path: self.path
+      }).done(function (data) {
+        var entriesWithNavOptMeta = [];
+        if (self.isDatabase() && data.top_tables) {
+          data.top_tables.forEach(function (topTable) {
+            var entry = self.dataCatalog.getEntry({ path: self.path.concat(topTable.name.toLowerCase()) });
+            entry.navOptMeta = topTable;
+            entriesWithNavOptMeta.push(entry);
           });
-          deferred.resolve(children);
+        } else if (self.isTableOrView() && data.values && data.values.selectColumns) {
+          data.values.selectColumns.forEach(function (selectColumn) {
+            var entry = self.dataCatalog.getEntry({ path: self.path.concat(selectColumn.columnName.toLowerCase().split('.')) });
+            entry.navOptMeta = selectColumn;
+            entriesWithNavOptMeta.push(entry);
+          })
         }
-      }).fail(deferred.reject);
-    });
-    return deferred.promise();
+        deferred.resolve(entriesWithNavOptMeta);
+      }).fail(deferred.reject));
+    } else {
+      deferred.resolve([]);
+    }
+    self.navOptMetaForChildrenPromise = new CancellablePromise(deferred.promise(), undefined, cancellablePromises);
+    return self.navOptMetaForChildrenPromise;
   };
 
   DataCatalogEntry.prototype.getKnownComment = function () {
@@ -229,19 +291,21 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   *
+   * @return {CancellablePromise}
    */
   DataCatalogEntry.prototype.getComment = function (apiOptions) {
     var self = this;
     var deferred = $.Deferred();
+    var cancellablePromises = [];
 
     var resolveWithSourceMeta = function () {
       if (self.sourceMeta) {
         deferred.resolve(self.sourceMeta && self.sourceMeta.comment || '');
       } else {
-        self.getSourceMeta(apiOptions).done(function (sourceMeta) {
+        cancellablePromises.push(self.getSourceMeta(apiOptions).done(function (sourceMeta) {
           deferred.resolve(sourceMeta && sourceMeta.comment || '');
-        }).fail(deferred.reject);
+        }).fail(deferred.reject));
       }
     };
 
@@ -249,19 +313,19 @@ var DataCatalog = (function () {
       if (self.navigatorMeta) {
         deferred.resolve(self.navigatorMeta.description || self.navigatorMeta.originalDescription || '');
       } else {
-        self.getNavigatorMeta(apiOptions).done(function (navigatorMeta) {
+        cancellablePromises.push(self.getNavigatorMeta(apiOptions).done(function (navigatorMeta) {
           if (navigatorMeta) {
             deferred.resolve(navigatorMeta.description || navigatorMeta.originalDescription || '');
           } else {
             resolveWithSourceMeta();
           }
-        }).fail(resolveWithSourceMeta)
+        }).fail(resolveWithSourceMeta))
       }
     } else {
       resolveWithSourceMeta();
     }
 
-    return deferred.promise();
+    return new CancellablePromise(deferred.promise(), undefined, cancellablePromises);
   };
 
   /**
@@ -270,6 +334,7 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
+   *
    * @return {Promise}
    */
   DataCatalogEntry.prototype.setComment = function (comment, apiOptions) {
@@ -433,7 +498,7 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   * @return {CancellablePromise}
    */
   DataCatalogEntry.prototype.getSourceMeta = function (apiOptions) {
     var self = this;
@@ -445,7 +510,7 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   * @return {CancellablePromise}
    */
   DataCatalogEntry.prototype.getNavigatorMeta = function (apiOptions) {
     var self = this;
@@ -457,16 +522,13 @@ var DataCatalog = (function () {
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
-   * @return {Promise}
+   * @return {CancellablePromise}
    */
   DataCatalogEntry.prototype.getSample = function () {
     var self = this;
     return self.lastSamplePromise || reloadSample(self, options)
   };
 
-
-
-
   var instances = {};
 
   /**