فهرست منبع

HUE-7820 [frontend] Use IndexedDB instead of TotalStorage for the DataCatalog

Johan Ahlen 7 سال پیش
والد
کامیت
557071a

+ 79 - 60
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -1532,8 +1532,6 @@ var ApiHelper = (function () {
    * @param {Object} options
    * @param {string} options.sourceType
    * @param {boolean} [options.silenceErrors]
-   * @param {boolean} [options.cachedOnly] - Default false
-   * @param {boolean} [options.refreshCache] - Default false
    *
    * @param {string[]} [options.path] - The path to fetch
    *
@@ -1543,19 +1541,35 @@ var ApiHelper = (function () {
     var self = this;
     var deferred = $.Deferred();
 
-    var request = fetchAssistData.bind(self)({
+    var request = $.ajax({
+      type: 'POST',
       url: AUTOCOMPLETE_API_PREFIX + options.path.join('/'),
-      sourceType: options.sourceType,
+      data: {
+        notebook: {},
+        snippet: ko.mapping.toJSON({
+          type: options.sourceType
+        })
+      },
+      timeout: options.timeout
+    }).success(function (data) {
+      data.notFound = data.status === 0 && data.code === 500 && data.error && (data.error.indexOf('Error 10001') !== -1 || data.error.indexOf('AnalysisException') !== -1);
+      data.hueTimestamp = Date.now();
+
+      // TODO: Display warning in autocomplete when an entity can't be found
+      // Hive example: data.error: [...] SemanticException [Error 10001]: Table not found default.foo
+      // Impala example: data.error: [...] AnalysisException: Could not resolve path: 'default.foo'
+      if (!data.notFound && self.successResponseIsError(data)) {
+        self.assistErrorCallback({
+          silenceErrors: options.silenceErrors,
+          errorCallback: deferred.reject
+        })(data);
+      } else {
+        deferred.resolve(data);
+      }
+    }).fail(self.assistErrorCallback({
       silenceErrors: options.silenceErrors,
-      cachedOnly: options.cachedOnly,
-      refreshCache: options.refreshCache,
-      successCallback: deferred.resolve,
-      errorCallback: self.assistErrorCallback({
-        errorCallback: deferred.reject,
-        silenceErrors: options.silenceErrors
-      }),
-      cacheCondition: genericCacheCondition
-    });
+      errorCallback: deferred.reject
+    }));
 
     return new CancellablePromise(deferred.promise(), request);
   };
@@ -1606,8 +1620,6 @@ var ApiHelper = (function () {
    *
    * @param {Object} options
    * @param {boolean} [options.silenceErrors]
-   * @param {boolean} [options.noCache]
-   * @param {boolean} [options.refreshCache] - Default false
    *
    * @param {string} options.sourceType
    * @param {string[]} options.path
@@ -1617,19 +1629,20 @@ var ApiHelper = (function () {
   ApiHelper.prototype.fetchSample = function (options) {
     var self = this;
     var deferred = $.Deferred();
-    var request = fetchAssistData.bind(self)({
-      url: SAMPLE_API_PREFIX + options.path.join('/'),
-      sourceType: options.sourceType,
-      noCache: options.noCache,
-      refreshCache: options.refreshCache,
+
+    var request = self.simplePost(SAMPLE_API_PREFIX + options.path.join('/'), {
+      notebook: {},
+      snippet: ko.mapping.toJSON({
+        type: options.sourceType
+      })
+    }, {
       silenceErrors: options.silenceErrors,
-      successCallback: deferred.resolve,
-      errorCallback: self.assistErrorCallback({
-        errorCallback: deferred.reject,
-        silenceErrors: options.silenceErrors
-      }),
-      cacheCondition: genericCacheCondition
-    });
+      successCallback: function (data) {
+        data.hueTimestamp = Date.now();
+        deferred.resolve(data);
+      }
+    }).done().fail(deferred.reject);
+
     return new CancellablePromise(deferred.promise(), request);
   };
 
@@ -1638,8 +1651,6 @@ var ApiHelper = (function () {
    *
    * @param {Object} options
    * @param {boolean} [options.silenceErrors]
-   * @param {boolean} [options.noCache]
-   * @param {boolean} [options.refreshCache] - Default false
    *
    * @param {boolean} [options.isView] - Default false
    * @param {string[]} options.path
@@ -1659,20 +1670,19 @@ var ApiHelper = (function () {
       url +=  '?type=field&database=' + options.path[0] + '&table=' + options.path[1] + '&name=' + options.path[2];
     }
 
-    var request = fetchAssistData.bind(self)({
-      url: url,
-      sourceType: 'nav',
-      noCache: options.noCache,
-      refreshCache: options.refreshCache,
+    var request = self.simplePost(url, {
+      notebook: {},
+      snippet: ko.mapping.toJSON({
+        type: 'nav'
+      })
+    }, {
       silenceErrors: options.silenceErrors,
       successCallback: function (data) {
-        deferred.resolve(data.entity || data);
+        data = data.entity || data;
+        data.hueTimestamp = Date.now();
+        deferred.resolve(data);
       },
-      errorCallback: self.assistErrorCallback({
-        errorCallback: deferred.reject,
-        silenceErrors: options.silenceErrors
-      }),
-      cacheCondition: genericCacheCondition
+      errorCallback: deferred.reject
     });
 
     return new CancellablePromise(deferred.promise(), request);
@@ -1706,23 +1716,39 @@ var ApiHelper = (function () {
    *
    * @param {Object} options
    * @param {boolean} [options.silenceErrors]
-   * @param {boolean} [options.refreshCache] - Default false
    * @param {string[]} options.path
    * @return {CancellablePromise}
    */
   ApiHelper.prototype.fetchNavOptMetadata = function (options) {
     var self = this;
     var deferred = $.Deferred();
-    var url = options.path.length === 1 ? NAV_OPT_URLS.TOP_TABLES : NAV_OPT_URLS.TOP_COLUMNS;
-    return self.fetchNavOptCached(url, {
-      path: options.path,
-      refreshCache: options.refreshCache,
+    var url, data;
+
+    if (options.path.length === 1) {
+      url = NAV_OPT_URLS.TOP_TABLES;
+      data = {
+        database: options.path[0]
+      };
+    } else if (options.path.length > 1) {
+      url = NAV_OPT_URLS.TOP_COLUMNS;
+      data = {
+        dbTables: ko.mapping.toJSON([options.path.join('.')])
+      };
+    } else {
+      deferred.reject();
+      return new CancellablePromise(deferred.promise());
+    }
+
+    var request = self.simplePost(url, data, {
       silenceErrors: options.silenceErrors,
-      successCallback: deferred.resolve,
+      successCallback: function (data) {
+        data.hueTimestamp = Date.now();
+        deferred.resolve(data);
+      },
       errorCallback: deferred.reject
-    }, function (data) {
-      return data.status === 0;
     });
+
+    return new CancellablePromise(deferred.promise(), request);
   };
 
 
@@ -2206,24 +2232,17 @@ var ApiHelper = (function () {
     var self = this;
     var deferred = $.Deferred();
 
-    var request = $.post(SEARCH_API, {
+    var request = self.simplePost(SEARCH_API, {
       query_s: ko.mapping.toJSON(options.query),
       limit: options.limit || 100,
       raw_query: !!options.rawQuery,
       sources: options.sources ? ko.mapping.toJSON(options.sources) : '["sql"]'
-	  }).done(function (data) {
-      if (data.status === 0 && !self.successResponseIsError(data)) {
-        deferred.resolve(data);
-      } else {
-        self.assistErrorCallback({
-          silenceErrors: options.silenceErrors,
-          errorCallback: deferred.reject
-        })(data);
-      }
-    }).fail(self.assistErrorCallback({
+    }, {
       silenceErrors: options.silenceErrors,
+      successCallback: deferred.resolve,
       errorCallback: deferred.reject
-    }));
+    });
+
     return new CancellablePromise(deferred.promise(), request);
   };
 

+ 19 - 31
desktop/core/src/desktop/static/desktop/js/assist/assistDbEntry.js

@@ -306,12 +306,10 @@ var AssistDbEntry = (function () {
 
     var successCallback = function(sourceMeta) {
       self.entries([]);
-      self.hasErrors(false);
-      self.loading(false);
-      self.loaded = true;
-
-
       self.catalogEntry.getChildren().done(function (catalogEntries) {
+        self.hasErrors(false);
+        self.loading(false);
+        self.loaded = true;
         var newEntries = [];
         catalogEntries.forEach(function (catalogEntry) {
           newEntries.push(self.createEntry(catalogEntry));
@@ -330,7 +328,9 @@ var AssistDbEntry = (function () {
         }
       });
 
-      self.catalogEntry.loadNavigatorMetaForChildren({ silenceErrors: self.navigationSettings.rightAssist });
+      if (self.assistDbSource.sourceType !== 'solr') {
+        self.catalogEntry.loadNavigatorMetaForChildren({ silenceErrors: self.navigationSettings.rightAssist });
+      }
     };
 
     var errorCallback = function () {
@@ -341,33 +341,21 @@ var AssistDbEntry = (function () {
     };
 
     if (!self.navigationSettings.rightAssist && HAS_OPTIMIZER && self.catalogEntry.isTable() && self.assistDbSource.sourceType !== 'solr') {
-      self.assistDbSource.apiHelper.fetchNavOptTopColumns({
-        sourceType: self.assistDbSource.sourceType,
-        successCallback: function (data) {
-          if (data.status === 0 && data.values && data.values.selectColumns && data.values.selectColumns.length > 0) {
-            var colIndex = {};
-            data.values.selectColumns.forEach(function (col) {
-              colIndex[col.columnName] = col;
-            });
-            $.when(loadEntriesDeferred).done(function () {
-              if (!self.hasErrors()) {
-                self.entries().forEach(function (entry) {
-                  if (colIndex[entry.catalogEntry.name]) {
-                    entry.popularity(colIndex[entry.catalogEntry.name].columnCount);
-                  }
-                });
-
-                if (self.assistDbSource.activeSort() === 'popular') {
-                  self.entries.sort(self.sortFunctions.popular);
-                }
+      self.catalogEntry.loadNavOptMetaForChildren({ silenceErrors: true }).done(function () {
+        loadEntriesDeferred.done(function () {
+          if (!self.hasErrors()) {
+            self.entries().forEach(function (entry) {
+              if (entry.catalogEntry.navOptMeta) {
+                entry.popularity(entry.catalogEntry.navOptMeta.columnCount)
               }
-            })
+            });
+
+            if (self.assistDbSource.activeSort() === 'popular') {
+              self.entries.sort(self.sortFunctions.popular);
+            }
           }
-        },
-        silenceErrors: true,
-        tables: [{ identifierChain: [{ name: self.catalogEntry.name }] }],
-        defaultDatabase: self.parent.catalogEntry.name
-      })
+        })
+      });
     }
 
     self.catalogEntry.getSourceMeta({ silenceErrors: self.navigationSettings.rightAssist }).done(successCallback).fail(errorCallback);

+ 48 - 52
desktop/core/src/desktop/static/desktop/js/assist/assistDbSource.js

@@ -122,44 +122,32 @@ var AssistDbSource = (function () {
     self.selectedDatabase.subscribe(function () {
       var db = self.selectedDatabase();
       if (HAS_OPTIMIZER && db && !db.popularityIndexSet && self.sourceType !== 'solr') {
-        self.apiHelper.fetchNavOptTopTables({
-          sourceType: self.sourceType,
-          database: db.catalogEntry.name,
-          silenceErrors: true,
-          timeout: AUTOCOMPLETE_TIMEOUT,
-          successCallback: function (data) {
-            var popularityIndex = {};
-            data.top_tables.forEach(function (topTable) {
-              if (topTable.popularity >= 5) {
-                popularityIndex[topTable.name] = topTable.popularity;
+        db.catalogEntry.loadNavOptMetaForChildren({ silenceErrors: true }).done(function () {
+          var applyPopularity = function () {
+            db.entries().forEach(function (entry) {
+              if (entry.catalogEntry.navOptMeta && entry.catalogEntry.navOptMeta.popularity >= 5) {
+                entry.popularity(entry.catalogEntry.navOptMeta.popularity )
               }
             });
-            var applyPopularity = function () {
-              db.entries().forEach(function (entry) {
-                if (popularityIndex[entry.catalogEntry.name]) {
-                  entry.popularity(popularityIndex[entry.catalogEntry.name]);
-                }
-              });
-              if (self.activeSort() === 'popular') {
-                db.entries.sort(sortFunctions.popular);
-              }
-            };
+            if (self.activeSort() === 'popular') {
+              db.entries.sort(sortFunctions.popular);
+            }
+          };
 
-            if (db.loading()) {
-              var subscription = db.loading.subscribe(function () {
+          if (db.loading()) {
+            var subscription = db.loading.subscribe(function () {
+              subscription.dispose();
+              applyPopularity();
+            });
+          } else if (db.entries().length == 0) {
+            var subscription = db.entries.subscribe(function (newEntries) {
+              if (newEntries.length > 0) {
                 subscription.dispose();
                 applyPopularity();
-              });
-            } else if (db.entries().length == 0) {
-              var subscription = db.entries.subscribe(function (newEntries) {
-                if (newEntries.length > 0) {
-                  subscription.dispose();
-                  applyPopularity();
-                }
-              });
-            } else {
-              applyPopularity();
-            }
+              }
+            });
+          } else {
+            applyPopularity();
           }
         });
       }
@@ -218,27 +206,35 @@ var AssistDbSource = (function () {
     var updateDatabases = function (names, lastSelectedDb) {
       dbIndex = {};
       var hasNavMeta = false;
-      var dbs = $.map(names, function(name) {
-        var catalogEntry = DataCatalog.getEntry({ sourceType: self.sourceType, path: name, definition: { type: 'database' }});
-        hasNavMeta = hasNavMeta || !!catalogEntry.navigatorMeta;
-        var database = new AssistDbEntry(catalogEntry, null, self, nestedFilter, self.i18n, self.navigationSettings);
-        dbIndex[name] = database;
-        if (name === lastSelectedDb) {
-          self.selectedDatabase(database);
-          self.selectedDatabaseChanged();
-        }
-        return database;
+      var dbPromises = [];
+      var dbs = [];
+
+      names.forEach(function (name) {
+        dbPromises.push(DataCatalog.getEntry({ sourceType: self.sourceType, path: name, definition: { type: 'database' }}).done( function (catalogEntry) {
+          hasNavMeta = hasNavMeta || !!catalogEntry.navigatorMeta;
+          var database = new AssistDbEntry(catalogEntry, null, self, nestedFilter, self.i18n, self.navigationSettings);
+          dbIndex[name] = database;
+          if (name === lastSelectedDb) {
+            self.selectedDatabase(database);
+            self.selectedDatabaseChanged();
+          }
+          dbs.push(database);
+        }));
       });
 
-      if (!hasNavMeta) {
-        DataCatalog.getEntry({ sourceType: self.sourceType, path: [], definition: { type: 'source' } }).loadNavigatorMetaForChildren({ silenceErrors: true });
-      }
-
-      dbs.sort(sortFunctions[self.activeSort()]);
-      self.databases(dbs);
-      self.reloading(false);
-      self.loading(false);
-      self.loaded(true);
+      $.when.apply($, dbPromises).always(function () {
+        if (!hasNavMeta) {
+          if (self.sourceType !== 'solr') {
+            DataCatalog.getEntry({ sourceType: self.sourceType, path: [], definition: { type: 'source' } })
+              .done(function (catalogEntry) { catalogEntry.loadNavigatorMetaForChildren({ silenceErrors: true }) });
+          }
+        }
+        dbs.sort(sortFunctions[self.activeSort()]);
+        self.databases(dbs);
+        self.reloading(false);
+        self.loading(false);
+        self.loaded(true);
+      });
     };
 
     self.setDatabase = function (databaseName) {

+ 207 - 86
desktop/core/src/desktop/static/desktop/js/dataCatalog.js

@@ -16,7 +16,6 @@
 
 var DataCatalog = (function () {
 
-
   /**
    * @param {String} sourceType
    * @constructor
@@ -25,8 +24,44 @@ var DataCatalog = (function () {
     var self = this;
     self.sourceType = sourceType;
     self.entries = {};
+    self.store = localforage.createInstance({
+      name: "HueDataCatalog_" + self.sourceType + '_' + LOGGED_USERNAME // TODO: Add flag for embedded mode
+    });
   }
 
+  DataCatalog.prototype.clearStorage = function () {
+    var self = this;
+    self.store.clear()
+  };
+
+  DataCatalog.prototype.updateStore = function (dataCatalogEntry) {
+    var self = this;
+    self.store.setItem(dataCatalogEntry.path.join('.'), {
+      definition: dataCatalogEntry.definition,
+      sourceMeta: dataCatalogEntry.sourceMeta,
+      sample: dataCatalogEntry.sample,
+      navigatorMeta: dataCatalogEntry.navigatorMeta,
+      navOptMeta: dataCatalogEntry.navOptMeta
+    });
+  };
+
+  var mergeFromStoreEntry = function (dataCatalogEntry, storeEntry) {
+    var mergeAttribute = function (attributeName, ttl, promiseName) {
+      if (storeEntry[attributeName] && (!storeEntry[attributeName].hueTimestamp || (Date.now() - storeEntry[attributeName].hueTimestamp) < ttl)) {
+        dataCatalogEntry[attributeName] = storeEntry[attributeName];
+        if (promiseName) {
+          dataCatalogEntry[promiseName] = $.Deferred().resolve(dataCatalogEntry[attributeName]).promise();
+        }
+      }
+    };
+
+    mergeAttribute('definition', CACHEABLE_TTL.default);
+    mergeAttribute('sourceMeta', CACHEABLE_TTL.default, 'sourceMetaPromise');
+    mergeAttribute('sample', CACHEABLE_TTL.default, 'samplePromise');
+    mergeAttribute('navigatorMeta', CACHEABLE_TTL.default, 'navigatorMetaPromise');
+    mergeAttribute('navOptMeta', CACHEABLE_TTL.optimizer);
+  };
+
   /**
    * @param {Object} options
    * @param {string|string[]} options.path
@@ -35,17 +70,44 @@ var DataCatalog = (function () {
    */
   DataCatalog.prototype.getEntry = function (options) {
     var self = this;
+    var deferred = $.Deferred();
     var identifier = typeof options.path === 'string' ? options.path : (typeof options.path === 'string' ? options.path : options.path.join('.'));
-    return self.entries[identifier] || (self.entries[identifier] = new DataCatalogEntry(self, options.path, options.definition));
+    if (self.entries[identifier]) {
+      deferred.resolve(self.entries[identifier]);
+    } else {
+      deferred.done(function (entry) {
+        self.entries[identifier] = entry;
+      });
+      self.store.getItem(identifier).then(function (storeEntry) {
+        var definition = storeEntry ? storeEntry.definition : options.definition;
+        if (self.entries[identifier]) {
+          deferred.resolve(self.entries[identifier]);
+          return;
+        }
+        var entry = new DataCatalogEntry(self, options.path, definition);
+        if (storeEntry) {
+          mergeFromStoreEntry(entry, storeEntry);
+        } else {
+          entry.saveLater();
+        }
+        deferred.resolve(entry);
+      }).catch(function (error) {
+        console.warn(error);
+        var entry = new DataCatalogEntry(self, options.path, definition);
+        entry.saveLater();
+        deferred.resolve(entry);
+      })
+    }
+    return deferred.promise();
   };
 
   var fetchMeta = function (apiHelperFunction, dataCatalogEntry, apiOptions) {
     return ApiHelper.getInstance()[apiHelperFunction]({
       sourceType: dataCatalogEntry.dataCatalog.sourceType,
       path: dataCatalogEntry.path,
-      silenceErrors: apiOptions && apiOptions.silenceErrors,
-      cachedOnly: apiOptions && apiOptions.cachedOnly,
-      refreshCache: apiOptions && apiOptions.refreshCache
+      silenceErrors: apiOptions && apiOptions.silenceErrors
+    }).done(function () {
+      dataCatalogEntry.saveLater();
     })
   };
 
@@ -59,11 +121,11 @@ var DataCatalog = (function () {
    * @return {CancellablePromise}
    */
   var reloadSourceMeta = function (dataCatalogEntry, apiOptions) {
-    dataCatalogEntry.lastSourceMetaPromise = fetchMeta('fetchSourceMetadata', dataCatalogEntry, apiOptions);
-    dataCatalogEntry.lastSourceMetaPromise.done(function (sourceMeta) {
+    dataCatalogEntry.sourceMetaPromise = fetchMeta('fetchSourceMetadata', dataCatalogEntry, apiOptions);
+    dataCatalogEntry.sourceMetaPromise.done(function (sourceMeta) {
       dataCatalogEntry.sourceMeta = sourceMeta;
     });
-    return dataCatalogEntry.lastSourceMetaPromise;
+    return dataCatalogEntry.sourceMetaPromise;
   };
 
   /**
@@ -76,15 +138,15 @@ var DataCatalog = (function () {
    * @return {CancellablePromise}
    */
   var reloadNavigatorMeta = function (dataCatalogEntry, apiOptions) {
-    if (HAS_NAVIGATOR) {
-      dataCatalogEntry.lastNavigatorMetaPromise = fetchMeta('fetchNavigatorMetadata', dataCatalogEntry, apiOptions);
-      dataCatalogEntry.lastNavigatorMetaPromise.done(function (navigatorMeta) {
+    if (HAS_NAVIGATOR && (dataCatalogEntry.dataCatalog.sourceType === 'hive' || dataCatalogEntry.dataCatalog.sourceType === 'impala')) {
+      dataCatalogEntry.navigatorMetaPromise = fetchMeta('fetchNavigatorMetadata', dataCatalogEntry, apiOptions);
+      dataCatalogEntry.navigatorMetaPromise.done(function (navigatorMeta) {
         dataCatalogEntry.navigatorMeta = navigatorMeta;
       });
     } else {
-      dataCatalogEntry.lastNavigatorMetaPromise =  $.Deferred.reject().promise();
+      dataCatalogEntry.navigatorMetaPromise =  $.Deferred.reject().promise();
     }
-    return dataCatalogEntry.lastNavigatorMetaPromise;
+    return dataCatalogEntry.navigatorMetaPromise;
   };
 
   /**
@@ -97,11 +159,11 @@ var DataCatalog = (function () {
    * @return {CancellablePromise}
    */
   var reloadSample = function (dataCatalogEntry, apiOptions) {
-    dataCatalogEntry.lastSamplePromise = fetchMeta('fetchSample', dataCatalogEntry, apiOptions);
-    dataCatalogEntry.lastSamplePromise.done(function (sample) {
+    dataCatalogEntry.samplePromise = fetchMeta('fetchSample', dataCatalogEntry, apiOptions);
+    dataCatalogEntry.samplePromise.done(function (sample) {
       dataCatalogEntry.sample = sample;
     });
-    return dataCatalogEntry.lastSamplePromise;
+    return dataCatalogEntry.samplePromise;
   };
 
   /**
@@ -119,69 +181,85 @@ var DataCatalog = (function () {
 
     self.definition = definition;
 
-    self.lastSourceMetaPromise = undefined;
+    self.sourceMetaPromise = undefined;
     self.sourceMeta = undefined;
 
-    self.lastNavigatorMetaPromise = undefined;
+    self.navigatorMetaPromise = undefined;
     self.navigatorMeta = undefined;
-    self.navigatorMetaForChildrenPromise = undefined;
 
-    self.lastSamplePromise = undefined;
+    self.samplePromise = undefined;
     self.sample = undefined;
 
     self.navOptMeta = undefined;
+
+    self.navigatorMetaForChildrenPromise = undefined;
     self.navOptMetaForChildrenPromise = undefined;
 
-    self.children = undefined;
+    self.childrenPromise = undefined;
+    self.saveTimeout = -1;
   }
 
+  DataCatalogEntry.prototype.saveLater = function () {
+    var self = this;
+    window.clearTimeout(self.saveTimeout);
+    self.saveTimeout = window.setTimeout(function () {
+      self.dataCatalog.updateStore(self);
+    }, 1000);
+  };
+
   /**
    * @param {Object} [apiOptions]
    * @param {boolean} [apiOptions.silenceErrors]
    * @param {boolean} [apiOptions.cachedOnly]
    * @param {boolean} [apiOptions.refreshCache]
    *
-   * @return {CancellablePromise}
+   * @return {Promise}
    */
   DataCatalogEntry.prototype.getChildren = function (apiOptions) {
     var self = this;
+    if (self.childrenPromise && (!apiOptions || !apiOptions.refreshCache)) {
+      return self.childrenPromise;
+    }
     var deferred = $.Deferred();
-    var cancellablePromises = [];
-    if (self.children) {
-      deferred.resolve(self.children);
-    } else {
-      self.getSourceMeta(apiOptions).done(function (sourceMeta) {
-        if (self.children) {
-          deferred.resolve(self.children);
-        } else {
-          self.children = [];
-          var index = 0;
-          var entities = sourceMeta.databases || sourceMeta.tables_meta || sourceMeta.extended_columns || sourceMeta.fields;
-          if (entities) {
-            entities.forEach(function (entity) {
-              var catalogEntry = self.dataCatalog.getEntry({ path: self.path.concat(entity.name || entity) });
+    self.childrenPromise = deferred.promise();
+    self.getSourceMeta(apiOptions).done(function (sourceMeta) {
+      var promises = [];
+      var index = 0;
+      var entities = sourceMeta.databases || sourceMeta.tables_meta || sourceMeta.extended_columns || sourceMeta.fields;
+      if (entities) {
+        entities.forEach(function (entity) {
+          promises.push(self.dataCatalog.getEntry({ path: self.path.concat(entity.name || entity) }).done(function (catalogEntry) {
+            if (!catalogEntry.definition || typeof catalogEntry.definition.index === 'undefined') {
               var definition = typeof entity === 'object' ? entity : {};
+              if (typeof entity !== 'object' && self.path.length === 0) {
+                definition.type = 'database';
+              }
               definition.index = index++;
-              catalogEntry.definition = definition; // Override any existing definition as parent might know more
-              self.children.push(catalogEntry);
-            });
-          } else {
-            (sourceMeta.type === 'map' ? ['key', 'value'] : ['item']).forEach(function (path) {
-              if (sourceMeta[path]) {
-                var catalogEntry = self.dataCatalog.getEntry({ path: self.path.concat(path) });
+              catalogEntry.definition = definition;
+              catalogEntry.saveLater();
+            }
+          }));
+        });
+      } else {
+        (sourceMeta.type === 'map' ? ['key', 'value'] : ['item']).forEach(function (path) {
+          if (sourceMeta[path]) {
+            promises.push(self.dataCatalog.getEntry({ path: self.path.concat(path) }).done(function (catalogEntry) {
+              if (!catalogEntry.definition || typeof catalogEntry.definition.index === 'undefined') {
                 var definition = sourceMeta[path];
                 definition.index = index++;
                 definition.isMapValue = path === 'value';
-                catalogEntry.definition = definition; // Override any existing definition as parent might know more
-                self.children.push(catalogEntry);
+                catalogEntry.definition = definition;
+                catalogEntry.saveLater();
               }
-            })
+            }));
           }
-          deferred.resolve(self.children);
-        }
-      })
-    }
-    return deferred.promise();
+        })
+      }
+      $.when.apply($, promises).done(function () {
+        deferred.resolve(Array.prototype.slice.call(arguments));
+      });
+    }).fail(deferred.reject);
+    return self.childrenPromise;
   };
 
   /**
@@ -194,6 +272,10 @@ var DataCatalog = (function () {
   DataCatalogEntry.prototype.loadNavigatorMetaForChildren = function (apiOptions) {
     var self = this;
 
+    if (self.dataCatalog.sourceType !== 'hive' && self.dataCatalog.sourceType !== 'impala') {
+      return $.Deferred().reject().promise();
+    }
+
     if (self.navigatorMetaForChildrenPromise && (!apiOptions || !apiOptions.refreshCache)) {
       return self.navigatorMetaForChildrenPromise;
     }
@@ -217,16 +299,23 @@ var DataCatalog = (function () {
         limit: children.length,
         silenceErrors: apiOptions && apiOptions.silenceErrors
       }).done(function (result) {
-        var entriesWithNavMeta = [];
         if (result && result.entities && result.entities.length > 0) {
+          var entryPromises = [];
           result.entities.forEach(function (entity) {
-            var entry = self.dataCatalog.getEntry({ path: self.path.concat((entity.name || entity.originalName).toLowerCase())});
-            entry.navigatorMeta = entity;
-            entry.lastNavigatorMetaPromise = $.Deferred().resolve(entry.navigatorMeta).promise();
-            entriesWithNavMeta.push(entry);
+            var entryPromise = self.dataCatalog.getEntry({ path: self.path.concat((entity.name || entity.originalName).toLowerCase())}).done(function(catalogEntry) {
+              catalogEntry.navigatorMeta = entity;
+              catalogEntry.navigatorMetaPromise = $.Deferred().resolve(catalogEntry.navigatorMeta).promise();
+              catalogEntry.saveLater();
+            });
+            entryPromises.push(entryPromise);
+            cancellablePromises.push(entryPromise);
           });
+          $.when.apply($, entryPromises).done(function () {
+            deferred.resolve(Array.prototype.slice.call(arguments));
+          });
+        } else {
+          deferred.resolve([]);
         }
-        deferred.resolve(entriesWithNavMeta);
       }).fail(deferred.reject));
     }).fail(deferred.reject));
 
@@ -244,6 +333,9 @@ var DataCatalog = (function () {
    */
   DataCatalogEntry.prototype.loadNavOptMetaForChildren = function (apiOptions) {
     var self = this;
+    if (self.dataCatalog.sourceType !== 'hive' && self.dataCatalog.sourceType !== 'impala') {
+      return $.Deferred().reject().promise();
+    }
     if (self.navOptMetaForChildrenPromise && (!apiOptions || !apiOptions.refreshCache)) {
       return self.navOptMetaForChildrenPromise;
     }
@@ -255,21 +347,29 @@ var DataCatalog = (function () {
         refreshCache: apiOptions && apiOptions.refreshCache,
         path: self.path
       }).done(function (data) {
-        var entriesWithNavOptMeta = [];
+        var entryPromises = [];
         if (self.isDatabase() && data.top_tables) {
           data.top_tables.forEach(function (topTable) {
-            var entry = self.dataCatalog.getEntry({ path: self.path.concat(topTable.name.toLowerCase()) });
-            entry.navOptMeta = topTable;
-            entriesWithNavOptMeta.push(entry);
+            var entryPromise = self.dataCatalog.getEntry({ path: self.path.concat(topTable.name.toLowerCase()) }).done(function (entry) {
+              entry.navOptMeta = topTable;
+              entry.saveLater();
+            });
+            entryPromises.push(entryPromise);
+            cancellablePromises.push(entryPromise);
           });
         } else if (self.isTableOrView() && data.values && data.values.selectColumns) {
           data.values.selectColumns.forEach(function (selectColumn) {
-            var entry = self.dataCatalog.getEntry({ path: self.path.concat(selectColumn.columnName.toLowerCase().split('.')) });
-            entry.navOptMeta = selectColumn;
-            entriesWithNavOptMeta.push(entry);
+            var entryPromise = self.dataCatalog.getEntry({ path: self.path.concat(selectColumn.columnName.toLowerCase().split('.')) }).done(function (entry) {
+              entry.navOptMeta = selectColumn;
+              entry.saveLater();
+            });
+            entryPromises.push(entryPromise);
+            cancellablePromises.push(entryPromise);
           })
         }
-        deferred.resolve(entriesWithNavOptMeta);
+        $.when.apply($, entryPromises).done(function () {
+          deferred.resolve(Array.prototype.slice.call(arguments));
+        });
       }).fail(deferred.reject));
     } else {
       deferred.resolve([]);
@@ -280,7 +380,7 @@ var DataCatalog = (function () {
 
   DataCatalogEntry.prototype.getKnownComment = function () {
     var self = this;
-    if (self.navigatorMeta) {
+    if (self.navigatorMeta && (self.dataCatalog.sourceType === 'hive' || self.dataCatalog.sourceType === 'impala')) {
       return self.navigatorMeta.description || self.navigatorMeta.originalDescription || ''
     }
     return self.sourceMeta && self.sourceMeta.comment || '';
@@ -309,7 +409,7 @@ var DataCatalog = (function () {
       }
     };
 
-    if (HAS_NAVIGATOR) {
+    if (HAS_NAVIGATOR && (self.dataCatalog.sourceType === 'hive' || self.dataCatalog.sourceType === 'impala')) {
       if (self.navigatorMeta) {
         deferred.resolve(self.navigatorMeta.description || self.navigatorMeta.originalDescription || '');
       } else {
@@ -341,7 +441,7 @@ var DataCatalog = (function () {
     var self = this;
     var deferred = $.Deferred();
 
-    if (HAS_NAVIGATOR) {
+    if (HAS_NAVIGATOR && (self.dataCatalog.sourceType === 'hive' || self.dataCatalog.sourceType === 'impala')) {
       self.getNavigatorMeta(apiOptions).done(function (navigatorMeta) {
         if (navigatorMeta) {
           ApiHelper.getInstance().updateNavigatorMetadata({
@@ -494,39 +594,60 @@ var DataCatalog = (function () {
   };
 
   /**
-   * @param {Object} [apiOptions]
-   * @param {boolean} [apiOptions.silenceErrors]
-   * @param {boolean} [apiOptions.cachedOnly]
-   * @param {boolean} [apiOptions.refreshCache]
+   * @param {Object} [options]
+   * @param {boolean} [options.silenceErrors]
+   * @param {boolean} [options.cachedOnly]
+   * @param {boolean} [options.refreshCache]
    * @return {CancellablePromise}
    */
-  DataCatalogEntry.prototype.getSourceMeta = function (apiOptions) {
+  DataCatalogEntry.prototype.getSourceMeta = function (options) {
     var self = this;
-    return self.lastSourceMetaPromise || reloadSourceMeta(self, apiOptions);
+    if (options && options.cachedOnly) {
+      return self.sourceMetaPromise || $.Deferred().reject(false).promise();
+    }
+    if (options && options.refreshCache) {
+      return reloadSourceMeta(self, options);
+    }
+    return self.sourceMetaPromise || reloadSourceMeta(self, options);
   };
 
   /**
-   * @param {Object} [apiOptions]
-   * @param {boolean} [apiOptions.silenceErrors]
-   * @param {boolean} [apiOptions.cachedOnly]
-   * @param {boolean} [apiOptions.refreshCache]
+   * @param {Object} [options]
+   * @param {boolean} [options.silenceErrors]
+   * @param {boolean} [options.cachedOnly]
+   * @param {boolean} [options.refreshCache]
    * @return {CancellablePromise}
    */
-  DataCatalogEntry.prototype.getNavigatorMeta = function (apiOptions) {
+  DataCatalogEntry.prototype.getNavigatorMeta = function (options) {
     var self = this;
-    return self.lastNavigatorMetaPromise || reloadNavigatorMeta(self, apiOptions)
+    if (self.dataCatalog.sourceType !== 'hive' && self.dataCatalog.sourceType !== 'impala') {
+      return $.Deferred().reject().promise();
+    }
+    if (options && options.cachedOnly) {
+      return self.navigatorMetaPromise || $.Deferred().reject(false).promise();
+    }
+    if (options && options.refreshCache) {
+      return reloadNavigatorMeta(self, options);
+    }
+    return self.navigatorMetaPromise || reloadNavigatorMeta(self, options)
   };
 
   /**
-   * @param {Object} [apiOptions]
-   * @param {boolean} [apiOptions.silenceErrors]
-   * @param {boolean} [apiOptions.cachedOnly]
-   * @param {boolean} [apiOptions.refreshCache]
+   * @param {Object} [options]
+   * @param {boolean} [options.silenceErrors]
+   * @param {boolean} [options.cachedOnly]
+   * @param {boolean} [options.refreshCache]
    * @return {CancellablePromise}
    */
-  DataCatalogEntry.prototype.getSample = function () {
+  DataCatalogEntry.prototype.getSample = function (options) {
     var self = this;
-    return self.lastSamplePromise || reloadSample(self, options)
+    if (options && options.cachedOnly) {
+      return self.samplePromise || $.Deferred().reject(false).promise();
+    }
+    if (options && options.refreshCache) {
+      return reloadSample(self, options);
+    }
+    return self.samplePromise || reloadSample(self, options)
   };
 
   var instances = {};

+ 85 - 68
desktop/core/src/desktop/templates/assist.mako

@@ -2579,7 +2579,15 @@ from desktop.views import _ko
 
         var updateOnVisible = false;
 
+        var runningPromises = [];
+
         var handleLocationUpdate = function (activeLocations) {
+          while (runningPromises.length) {
+            var promise = runningPromises.pop();
+            if (promise.cancel) {
+              promise.cancel();
+            }
+          }
           updateOnVisible = false;
           assistDbSource.sourceType = activeLocations.type;
           if (!activeLocations) {
@@ -2602,62 +2610,82 @@ from desktop.views import _ko
 
             activeLocations.activeStatementLocations.forEach(function (location) {
               if (location.type === 'table' && (location.identifierChain.length !== 1 || !ctes[location.identifierChain[0].name.toLowerCase()])) {
-                var database = location.identifierChain.length === 2 ? location.identifierChain[0].name : activeLocations.defaultDatabase;
-                database = database.toLowerCase();
-                var catalogEntry = DataCatalog.getEntry({ sourceType: activeLocations.type, path: [database], definition: { type: 'database' }});
-                if (!databaseIndex[database]) {
-                  databaseIndex[database] = new AssistDbEntry(
+                var tableDeferred = $.Deferred();
+                var dbDeferred = $.Deferred();
+                runningPromises.push(tableDeferred);
+                runningPromises.push(dbDeferred);
+
+                var qid = createQualifiedIdentifier(location.identifierChain, activeLocations.defaultDatabase);
+                if (activeTableIndex[qid]) {
+                  tableQidIndex[qid] = true;
+                  tableDeferred.resolve(activeTableIndex[qid]);
+                  dbDeferred.resolve(activeTableIndex[qid].parent);
+                } else {
+                  var database = location.identifierChain.length === 2 ? location.identifierChain[0].name : activeLocations.defaultDatabase;
+                  database = database.toLowerCase();
+                  if (databaseIndex[database]) {
+                    dbDeferred.resolve(databaseIndex[database]);
+                  } else {
                     DataCatalog.getEntry({
                       sourceType: activeLocations.type,
-                      path: [database],
+                      path: [ database ],
                       definition: { type: 'database' }
-                    }),
-                    null,
-                    assistDbSource,
-                    self.filter,
-                    i18n,
-                    navigationSettings
-                  );
-                }
-                var qid = createQualifiedIdentifier(location.identifierChain, activeLocations.defaultDatabase);
-                tableQidIndex[qid] = true;
-                if (!activeTableIndex[qid]) {
-                  var tableName = location.identifierChain[location.identifierChain.length - 1].name;
+                    }).done(function (catalogEntry) {
+                      databaseIndex[database] = new AssistDbEntry(catalogEntry, null, assistDbSource, self.filter, i18n,navigationSettings);
+                      updateTables = true;
+                      dbDeferred.resolve(databaseIndex[database])
+                    }).fail(function () {
+                      console.log('reject 1');
+                      dbDeferred.reject();
+                    });
+                  }
 
-                  activeTableIndex[createQualifiedIdentifier(location.identifierChain, activeLocations.defaultDatabase)] = new AssistDbEntry(
+                  dbDeferred.done(function (dbEntry) {
+                    var tableName = location.identifierChain[location.identifierChain.length - 1].name;
                     DataCatalog.getEntry({
                       sourceType: activeLocations.type,
                       path: [database, tableName],
                       definition: { type: 'table' }
-                    }),
-                    databaseIndex[database],
-                    assistDbSource,
-                    self.filter,
-                    i18n,
-                    navigationSettings
-                  );
-                  updateTables = true;
+                    }).done(function (catalogEntry) {
+                      var tableEntry = new AssistDbEntry(
+                              catalogEntry,
+                              dbEntry,
+                              assistDbSource,
+                              self.filter,
+                              i18n,
+                              navigationSettings
+                      );
+                      activeTableIndex[createQualifiedIdentifier(location.identifierChain, activeLocations.defaultDatabase)] = tableEntry;
+                      tableQidIndex[qid] = true;
+                      updateTables = true;
+                      tableDeferred.resolve(tableEntry)
+                    }).fail(tableDeferred.reject);
+                  }).fail(tableDeferred.reject);
                 }
               }
             });
-            Object.keys(activeTableIndex).forEach(function (key) {
-              if (!tableQidIndex[key]) {
-                delete activeTableIndex[key];
-                updateTables = true;
-              }
-            });
 
-            if (updateTables) {
-              var tables = [];
+            $.when.apply($, runningPromises).always(function () {
+              runningPromises.length = 0;
               Object.keys(activeTableIndex).forEach(function (key) {
-                tables.push(activeTableIndex[key]);
+                if (!tableQidIndex[key]) {
+                  delete activeTableIndex[key];
+                  updateTables = true;
+                }
               });
 
-              tables.sort(function (a, b) {
-                return a.catalogEntry.name.localeCompare(b.catalogEntry.name);
-              });
-              self.activeTables(tables);
-            }
+              if (updateTables) {
+                var tables = [];
+                Object.keys(activeTableIndex).forEach(function (key) {
+                  tables.push(activeTableIndex[key]);
+                });
+
+                tables.sort(function (a, b) {
+                  return a.catalogEntry.name.localeCompare(b.catalogEntry.name);
+                });
+                self.activeTables(tables);
+              }
+            });
           }
         };
 
@@ -2922,33 +2950,22 @@ from desktop.views import _ko
           });
 
           var fakeParentName = collectionName.indexOf('.') > -1 ? collectionName.split('.')[0] : 'default';
-          var assistFakeDb = new AssistDbEntry(
-              DataCatalog.getEntry({
-                sourceType: collection.engine(),
-                path: [fakeParentName],
-                definition: { type: 'database' }
-              }),
-              null,
-              assistDbSource,
-              self.filter,
-              i18n,
-              navigationSettings
-          );
-
-          var collectionEntry = new AssistDbEntry(
-              DataCatalog.getEntry({
-                sourceType: collection.engine(),
-                path: [fakeParentName, collectionName.indexOf('.') > -1 ? collectionName.split('.')[1] : collectionName],
-                definition: { type: 'table' }
-              }),
-              assistFakeDb,
-              assistDbSource,
-              self.filter,
-              i18n,
-              navigationSettings
-          );
-
-          self.activeTables([collectionEntry]);
+
+          DataCatalog.getEntry({
+            sourceType: collection.engine(),
+            path: [fakeParentName],
+            definition: { type: 'database' }
+          }).done(function (fakeDbCatalogEntry) {
+            var assistFakeDb = new AssistDbEntry(fakeDbCatalogEntry, null, assistDbSource, self.filter, i18n, navigationSettings);
+            DataCatalog.getEntry({
+              sourceType: collection.engine(),
+              path: [fakeParentName, collectionName.indexOf('.') > -1 ? collectionName.split('.')[1] : collectionName],
+              definition: { type: 'table' }
+            }).done(function (collectionCatalogEntry) {
+              var collectionEntry = new AssistDbEntry(collectionCatalogEntry, assistFakeDb, assistDbSource, self.filter, i18n, navigationSettings);
+              self.activeTables([collectionEntry]);
+            });
+          });
 
           self.autocompleteFromEntries = function (nonPartial, partial) {
             var added = {};

+ 5 - 0
desktop/core/src/desktop/templates/common_header.mako

@@ -218,6 +218,11 @@ if USE_NEW_EDITOR.get():
 
   <script type="text/javascript">
     $(document).ready(function () {
+      localforage.config({
+        version: 1.0,
+        storeName: 'hue_store',
+      });
+
       huePubSub.subscribe('get.current.app.name', function () {
         var appName = '';
         if ('${ 'metastore' in apps }' === 'True' && location.href.indexOf('${"metastore" in apps and apps["metastore"].display_name}') !== -1) {