Explorar o código

HUE-7820 [core] Add ability to load navigator metadata for all children of a sqlMetadata instance in one go

Johan Ahlen %!s(int64=7) %!d(string=hai) anos
pai
achega
f123bed

+ 24 - 39
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -68,6 +68,9 @@ var ApiHelper = (function () {
   var DASHBOARD_TERMS_API = '/dashboard/get_terms';
   var DASHBOARD_STATS_API = '/dashboard/get_stats';
 
+  var SEARCH_API = '/desktop/api/search/entities';
+  var INTERACTIVE_SEARCH_API = '/desktop/api/search/entities_interactive';
+
   var HBASE_API_PREFIX = '/hbase/api/';
   var SAVE_TO_FILE = '/filebrowser/save';
 
@@ -2127,7 +2130,7 @@ var ApiHelper = (function () {
 
   ApiHelper.prototype.fetchHueDocsInteractive = function (query) {
     var promise = $.Deferred();
-    $.post('/desktop/api/search/entities_interactive', {
+    $.post(INTERACTIVE_SEARCH_API, {
       query_s: ko.mapping.toJSON(query),
       limit: 5,
       sources: '["documents"]'
@@ -2143,7 +2146,7 @@ var ApiHelper = (function () {
 
   ApiHelper.prototype.fetchNavEntitiesInteractive = function (query) {
     var promise = $.Deferred();
-    $.post('/desktop/api/search/entities_interactive', {
+    $.post(INTERACTIVE_SEARCH_API, {
       query_s: ko.mapping.toJSON(query),
       limit: 5,
       sources: '["sql", "hdfs", "s3"]'
@@ -2161,12 +2164,12 @@ var ApiHelper = (function () {
     var self = this;
 
     $.when.apply($, [
-      $.post('/desktop/api/search/entities_interactive', {
+      $.post(INTERACTIVE_SEARCH_API, {
         query_s: ko.mapping.toJSON(options.query),
         limit: 5,
         sources: '["sql", "hdfs", "s3"]'
       }),
-      $.post('/desktop/api/search/entities_interactive', {
+      $.post(INTERACTIVE_SEARCH_API, {
           query_s: ko.mapping.toJSON(options.query),
           limit: 5,
           sources: '["documents"]'
@@ -2184,47 +2187,29 @@ var ApiHelper = (function () {
     }).fail(self.assistErrorCallback(options));
   };
 
-  ApiHelper.prototype.navSearchAutocomplete = function (options) {
+  ApiHelper.prototype.searchEntities = function (options) {
     var self = this;
-    return $.post('/desktop/api/search/entities_interactive', {
-      query_s: ko.mapping.toJSON(options.query),
-      limit: 10,
-      sources: '["' + options.source + '"]'
-    }).done(function (data) {
-      if (data.status === 0 && !self.successResponseIsError(data)) {
-        options.successCallback(data);
-      } else if (data.status === -2 && typeof data.message !== 'undefined' && typeof data.message.message !== 'undefined') {
-        options.errorCallback({
-          source: 'navigator',
-          message: data.message.message
-        })
-      } else if (data.status === -2 && typeof data.message !== 'undefined') {
-        options.errorCallback({
-          source: 'navigator',
-          message: data.message
-        })
-      } else {
-        self.assistErrorCallback(options)(data);
-      }
-    }).fail(self.assistErrorCallback(options));
-  };
+    var deferred = $.Deferred();
 
-  ApiHelper.prototype.navSearch = function (options) {
-    var self = this;
-    var promise = $.Deferred();
-
-    $.post('/desktop/api/search/entities', {
+    $.post(SEARCH_API, {
       query_s: ko.mapping.toJSON(options.query),
       limit: options.limit || 100,
-      sources: options.sources || '["sql"]'
-	}).done(function (data) {
-      if (data.status === 0) {
-        promise.resolve(data);
+      raw_query: !!options.rawQuery,
+      sources: options.sources ? ko.mapping.toJSON(options.sources) : '["sql"]'
+	  }).done(function (data) {
+      if (data.status === 0 && !self.successResponseIsError(data)) {
+        deferred.resolve(data);
       } else {
-        promise.reject(data);
+        self.assistErrorCallback({
+          silenceErrors: options.silenceErrors,
+          errorCallback: deferred.reject
+        })(data);
       }
-    }).fail(promise.reject);
-    return promise;
+    }).fail(self.assistErrorCallback({
+      silenceErrors: options.silenceErrors,
+      errorCallback: deferred.reject
+    }));
+    return deferred.promise();
   };
 
   ApiHelper.prototype.formatSql = function (statements) {

+ 25 - 23
desktop/core/src/desktop/static/desktop/js/assist/assistDbEntry.js

@@ -55,7 +55,9 @@ var AssistDbEntry = (function () {
 
     self.metadata = new SqlMetadata({
       sourceType: self.sourceType,
-      path: self.getHierarchy()
+      path: self.getHierarchy(),
+      silenceErrors: self.navigationSettings.rightAssist,
+      cachedOnly: false
     });
 
     self.filterColumnNames = ko.observable(false);
@@ -317,7 +319,7 @@ var AssistDbEntry = (function () {
 
     var loadEntriesDeferred = $.Deferred();
 
-    var successCallback = function(sqlMeta) {
+    var successCallback = function(sourceMeta) {
       self.entries([]);
       self.hasErrors(false);
       self.loading(false);
@@ -325,8 +327,8 @@ var AssistDbEntry = (function () {
 
       var newEntries = [];
       var index = 0;
-      if (typeof sqlMeta.sourceMeta.tables_meta !== "undefined") {
-        newEntries = $.map(sqlMeta.sourceMeta.tables_meta, function(table) {
+      if (typeof sourceMeta.tables_meta !== "undefined") {
+        newEntries = $.map(sourceMeta.tables_meta, function(table) {
           table.index = index++;
           table.title = table.name + (table.comment ? ' - ' + table.comment : '');
           table.displayName = table.name;
@@ -334,8 +336,8 @@ var AssistDbEntry = (function () {
           table.isView = /view/i.test(table.type);
           return self.createEntry(table);
         });
-      } else if (typeof sqlMeta.sourceMeta.extended_columns !== "undefined" && sqlMeta.sourceMeta.extended_columns !== null) {
-        newEntries = $.map(sqlMeta.sourceMeta.extended_columns, function (columnDef) {
+      } else if (typeof sourceMeta.extended_columns !== "undefined" && sourceMeta.extended_columns !== null) {
+        newEntries = $.map(sourceMeta.extended_columns, function (columnDef) {
           var displayName = columnDef.name;
           if (typeof columnDef.type !== "undefined" && columnDef.type !== null) {
             displayName += ' (' + columnDef.type + ')'
@@ -355,8 +357,8 @@ var AssistDbEntry = (function () {
           columnDef.type = shortType;
           return self.createEntry(columnDef);
         });
-      } else if (typeof sqlMeta.sourceMeta.columns !== "undefined" && sqlMeta.sourceMeta.columns !== null) {
-        newEntries = $.map(sqlMeta.sourceMeta.columns, function(columnName) {
+      } else if (typeof sourceMeta.columns !== "undefined" && sourceMeta.columns !== null) {
+        newEntries = $.map(sourceMeta.columns, function(columnName) {
           return self.createEntry({
             name: columnName,
             index: index++,
@@ -365,28 +367,28 @@ var AssistDbEntry = (function () {
             isColumn: true
           });
         });
-      } else if (sqlMeta.isMap()) {
+      } else if (sourceMeta.type === 'map') {
         newEntries = [
           self.createEntry({
             name: "key",
             index: index++,
-            displayName: "key (" + sqlMeta.sourceMeta.key.type + ")",
-            title: "key (" + sqlMeta.sourceMeta.key.type + ")",
-            type: sqlMeta.sourceMeta.key.type,
+            displayName: "key (" + sourceMeta.key.type + ")",
+            title: "key (" + sourceMeta.key.type + ")",
+            type: sourceMeta.key.type,
             isComplex: true
           }),
           self.createEntry({
             name: "value",
             index: index++,
-            displayName: "value (" + sqlMeta.sourceMeta.value.type + ")",
-            title: "value (" + sqlMeta.sourceMeta.value.type + ")",
+            displayName: "value (" + sourceMeta.value.type + ")",
+            title: "value (" + sourceMeta.value.type + ")",
             isMapValue: true,
-            type: sqlMeta.sourceMeta.value.type,
+            type: sourceMeta.value.type,
             isComplex: true
           })
         ];
-      } else if (sqlMeta.isStruct()) {
-        newEntries = $.map(sqlMeta.sourceMeta.fields, function(field) {
+      } else if (sourceMeta.type === 'struct') {
+        newEntries = $.map(sourceMeta.fields, function(field) {
           return self.createEntry({
             name: field.name,
             index: index++,
@@ -396,21 +398,21 @@ var AssistDbEntry = (function () {
             isComplex: true
           });
         });
-      } else if (sqlMeta.isArray()) {
+      } else if (sourceMeta.type === 'array') {
         newEntries = [
           self.createEntry({
             name: "item",
             index: index++,
-            displayName: "item (" + sqlMeta.sourceMeta.item.type + ")",
-            title: "item (" + sqlMeta.sourceMeta.item.type + ")",
+            displayName: "item (" + sourceMeta.item.type + ")",
+            title: "item (" + sourceMeta.item.type + ")",
             isArray: true,
-            type: sqlMeta.sourceMeta.item.type,
+            type: sourceMeta.item.type,
             isComplex: true
           })
         ];
       }
 
-      if (sqlMeta.isArray() || sqlMeta.isMap()) {
+      if (sourceMeta.type === 'array' || sourceMeta.type === 'map') {
         self.entries(newEntries);
         self.entries()[0].open(true);
       } else {
@@ -461,7 +463,7 @@ var AssistDbEntry = (function () {
       })
     }
 
-    self.metadata.loadSourceMeta(self.navigationSettings.rightAssist || !!silenceErrors, false).done(successCallback).fail(errorCallback);
+    self.metadata.getSourceMeta().done(successCallback).fail(errorCallback);
   };
 
   /**

+ 124 - 92
desktop/core/src/desktop/static/desktop/js/sqlMetadata.js

@@ -16,38 +16,143 @@
 
 var SqlMetadata = (function () {
 
+  var reloadSourceMeta = function (sqlMetadata, refreshCache) {
+    sqlMetadata.lastSourcePromise = $.Deferred();
+    ApiHelper.getInstance().fetchSourceMetadata({
+      sourceType: sqlMetadata.sourceType,
+      path: sqlMetadata.path,
+      silenceErrors: sqlMetadata.silenceErrors,
+      cachedOnly: sqlMetadata.cachedOnly,
+      refreshCache: refreshCache
+    }).done(function (data) {
+      sqlMetadata.lastSourcePromise.resolve(data);
+    }).fail(function (message) {
+      sqlMetadata.lastSourcePromise.reject(message);
+    });
+    return sqlMetadata.lastSourcePromise.promise();
+  };
+
+  var reloadNavigatorMeta = function (sqlMetadata) {
+    sqlMetadata.lastNavigatorPromise = $.Deferred();
+    if (HAS_NAVIGATOR) {
+      ApiHelper.getInstance().fetchNavigatorMetadata({
+        path: sqlMetadata.path,
+        silenceErrors: sqlMetadata.silenceErrors,
+      }).done(function (data) {
+        sqlMetadata.lastNavigatorPromise.resolve(data);
+      }).fail(sqlMetadata.lastNavigatorPromise.reject);
+    } else {
+      sqlMetadata.lastNavigatorPromise.reject();
+    }
+    return sqlMetadata.lastNavigatorPromise.promise();
+  };
+
   function SqlMetadata (options) {
     var self = this;
-    self.loaded = false;
-    self.hasErrors = false;
 
     self.sourceType = options.sourceType;
-    self.path = typeof options.path === 'string' ? options.path.split('.') : options.path;
-
-    self.sourceMeta = undefined;
-    self.navigatorMeta = undefined;
+    self.path = typeof options.path === 'string' && options.path ? options.path.split('.') : options.path || [];
 
+    self.partialSourceMeta = options.partialSourceMeta;
     self.lastSourcePromise = undefined;
     self.lastNavigatorPromise = undefined;
 
+    self.children = undefined;
+    self.lastNavigatorChildrenPromise = undefined;
+
     self.silenceErrors = options.silenceErrors;
     self.cachedOnly = options.cachedOnly;
   }
 
+  SqlMetadata.prototype.getChildren = function () {
+    var self = this;
+    var deferred = $.Deferred();
+    if (self.children) {
+      deferred.resolve(self.children);
+    } else {
+      self.getSourceMeta().done(function (sourceMeta) {
+        if (self.children) {
+          deferred.resolve(self.children);
+        } else {
+          self.children = [];
+          var entities = sourceMeta.databases || sourceMeta.tables_meta || sourceMeta.extended_columns || sourceMeta.fields;
+          if (entities) {
+            entities.forEach(function (entity) {
+              self.children.push(new SqlMetadata({
+                sourceType: self.sourceType,
+                path: self.path.concat(entity.name || entity),
+                silenceErrors: self.silenceErrors,
+                cachedOnly: self.cachedOnly,
+                partialSourceMeta: typeof entity === 'object' ? entity : undefined
+              }))
+            });
+          } else {
+            (sourceMeta.type === 'map' ? ['key', 'value'] : ['item']).forEach(function (path) {
+              if (sourceMeta[path]) {
+                self.children.push(new SqlMetadata({
+                  sourceType: self.sourceType,
+                  path: self.path.concat(path),
+                  silenceErrors: self.silenceErrors,
+                  cachedOnly: self.cachedOnly,
+                  partialSourceMeta: sourceMeta[path]
+                }));
+              }
+            })
+          }
+          deferred.resolve(self.children);
+        }
+      })
+    }
+    return deferred.promise();
+  };
+
+  SqlMetadata.prototype.loadNavigatorMetaForChildren = function () {
+    var self = this;
+    self.getChildren().done(function (children) {
+      var query;
+
+      // TODO: Add sourceType to nav search query
+      if (self.path.length) {
+        query = 'parentPath:"/' + self.path.join('/') + '" AND type:(table view field)';
+      } else {
+        query = 'type:database'
+      }
+
+      ApiHelper.getInstance().searchEntities({
+        query: query,
+        rawQuery: true,
+        limit: children.length
+      }).done(function (result) {
+        if (result && result.entities && result.entities.length > 0) {
+          var entityIndex = {};
+          result.entities.forEach(function (entity) {
+            entityIndex[entity.name || entity.originalName] = entity;
+          });
+          children.forEach(function (child) {
+            var name = child.path[child.path.length - 1];
+            if (entityIndex[name]) {
+              child.lastNavigatorPromise = $.Deferred().resolve(entityIndex[name]).promise();
+            }
+          });
+        }
+      })
+    })
+  };
+
   SqlMetadata.prototype.getComment = function () {
     var self = this;
     var deferred = $.Deferred();
 
     var resolveWithSourceMeta = function () {
-      self.getSourceMeta().done(function () {
-        deferred.resolve(self.sourceMeta && self.sourceMeta.comment || '');
-      });
+      self.getSourceMeta().done(function (sourceMeta) {
+        deferred.resolve(sourceMeta && sourceMeta.comment || '');
+      }).fail(deferred.reject);
     };
 
     if (HAS_NAVIGATOR) {
-      self.getNavigatorMeta().done(function () {
-        if (self.navigatorMeta && self.navigatorMeta.entity) {
-          deferred.resolve(self.navigatorMeta.entity.description || self.navigatorMeta.entity.originalDescription || '');
+      self.getNavigatorMeta().done(function (navigatorMeta) {
+        if (navigatorMeta && navigatorMeta.entity) {
+          deferred.resolve(navigatorMeta.entity.description || navigatorMeta.entity.originalDescription || '');
         } else {
           resolveWithSourceMeta();
         }
@@ -64,15 +169,15 @@ var SqlMetadata = (function () {
     var deferred = $.Deferred();
 
     if (HAS_NAVIGATOR) {
-      self.getNavigatorMeta().done(function () {
-        if (self.navigatorMeta && self.navigatorMeta.entity) {
+      self.getNavigatorMeta().done(function (navigatorMeta) {
+        if (navigatorMeta && navigatorMeta.entity) {
           ApiHelper.getInstance().updateNavigatorMetadata({
-            identity: self.navigatorMeta.entity.identity,
+            identity: navigatorMeta.entity.identity,
             properties: {
               description: comment
             }
           }).done(function () {
-            self.loadNavigatorMeta();
+            reloadNavigatorMeta(self);
             self.getComment().done(deferred.resolve);
           }).fail(deferred.reject);
         }
@@ -85,7 +190,7 @@ var SqlMetadata = (function () {
           comment: comment
         }
       }).done(function () {
-        self.loadSourceMeta(true);
+        reloadSourceMeta(self, true);
         self.getComment().done(deferred.resolve);
       }).fail(deferred.reject);
     }
@@ -95,85 +200,12 @@ var SqlMetadata = (function () {
 
   SqlMetadata.prototype.getSourceMeta = function () {
     var self = this;
-    return self.lastSourcePromise || self.loadSourceMeta()
+    return self.lastSourcePromise || reloadSourceMeta(self)
   };
 
   SqlMetadata.prototype.getNavigatorMeta = function () {
     var self = this;
-    return self.lastNavigatorPromise || self.loadNavigatorMeta()
-  };
-
-  SqlMetadata.prototype.isDatabase = function () {
-    var self = this;
-    return self.path.length === 1;
-  };
-
-  SqlMetadata.prototype.isTable = function () {
-    var self = this;
-    return self.sourceMeta && typeof self.sourceMeta.columns !== 'undefined' && !self.sourceMeta.is_view;
-  };
-
-  SqlMetadata.prototype.isView = function () {
-    var self = this;
-    return self.sourceMeta && typeof self.sourceMeta.columns !== 'undefined' && self.sourceMeta.is_view;
-  };
-
-  SqlMetadata.prototype.isField = function () {
-    var self = this;
-    return self.path.length > 2
-  };
-
-  SqlMetadata.prototype.isMap = function () {
-    var self = this;
-    return self.sourceMeta && self.sourceMeta.type === 'map';
-  };
-
-  SqlMetadata.prototype.isStruct = function () {
-    var self = this;
-    return self.sourceMeta && self.sourceMeta.type === 'struct';
-  };
-
-  SqlMetadata.prototype.isArray = function () {
-    var self = this;
-    return self.sourceMeta && self.sourceMeta.type === 'array';
-  };
-
-  SqlMetadata.prototype.loadSourceMeta = function (refreshCache) {
-    var self = this;
-    self.lastSourcePromise = $.Deferred();
-    ApiHelper.getInstance().fetchSourceMetadata({
-      sourceType: self.sourceType,
-      path: self.path,
-      silenceErrors: self.silenceErrors,
-      cachedOnly: self.cachedOnly,
-      refreshCache: refreshCache
-    }).done(function (data) {
-      self.sourceMeta = data;
-      self.loaded = true;
-      self.lastSourcePromise.resolve(self);
-    }).fail(function (message) {
-      self.hasErrors = true;
-      self.lastSourcePromise.reject(message);
-    });
-
-    return self.lastSourcePromise.promise();
-  };
-  
-  SqlMetadata.prototype.loadNavigatorMeta = function () {
-    var self = this;
-    self.lastNavigatorPromise = $.Deferred();
-    if (HAS_NAVIGATOR) {
-      ApiHelper.getInstance().fetchNavigatorMetadata({
-        path: self.path,
-        silenceErrors: self.silenceErrors,
-      }).done(function (data) {
-        self.navigatorMeta = data;
-        self.lastNavigatorPromise.resolve(self);
-      }).fail(self.lastNavigatorPromise.reject);
-    } else {
-      self.lastNavigatorPromise.resolve();
-    }
-    return self.lastNavigatorPromise.promise();
+    return self.lastNavigatorPromise || reloadNavigatorMeta(self)
   };
 
   return SqlMetadata;

+ 1 - 1
desktop/core/src/desktop/templates/assist.mako

@@ -2519,7 +2519,7 @@ from desktop.views import _ko
                 if (!table.loaded && !table.hasErrors() && !table.loading()) {
                   table.loadEntries(function () {
                     loadEntries(count);
-                  }, true);
+                  });
                   return false;
                 }
                 return !table.loading();