浏览代码

[assist] Add JS model for HDFS

Johan Ahlen 10 年之前
父节点
当前提交
e168f6c443

+ 103 - 0
desktop/core/src/desktop/static/desktop/js/assist/assistHdfsEntry.js

@@ -0,0 +1,103 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+(function (root, factory) {
+  if(typeof define === "function" && define.amd) {
+    define(['knockout'], factory);
+  } else {
+    root.AssistHdfsEntry = factory(ko);
+  }
+}(this, function (ko) {
+
+  /**
+   * @param {object} options
+   * @param {object} options.definition
+   * @param {string} options.definition.name
+   * @param {string} options.definition.type (file, dir)
+   * @param {AssistHdfsEntry} options.parent
+   * @param {AssistHelper} options.assistHelper
+   * @constructor
+   */
+  function AssistHdfsEntry (options) {
+    var self = this;
+
+    self.definition = options.definition;
+    self.assistHelper = options.assistHelper;
+    self.parent = options.parent;
+
+    self.entries = ko.observableArray([]);
+
+    self.loading = ko.observable(false);
+    self.open = ko.observable(false);
+
+    self.open.subscribe(function(newValue) {
+      if (newValue && self.entries().length == 0) {
+        self.loadEntries();
+      }
+    });
+
+    self.hasEntries = ko.computed(function() {
+      return self.entries().length > 0;
+    });
+  }
+
+  AssistHdfsEntry.prototype.loadEntries = function() {
+    var self = this;
+    if (self.loading()) {
+      return;
+    }
+    self.loading(true);
+
+    var successCallback = function(data) {
+      console.log(data);
+
+      self.entries($.map(data.files, function (file) {
+        return new AssistHdfsEntry({
+          definition: file,
+          parent: self,
+          assistHelper: self.assistHelper
+        })
+      }));
+      self.loading(false);
+    };
+
+    var errorCallback = function () {
+      console.log(data);
+      self.loading(false);
+    };
+
+    self.assistHelper.fetchHdfsPath(self.getHierarchy(), successCallback, errorCallback)
+  };
+
+  AssistHdfsEntry.prototype.getHierarchy = function () {
+    var self = this;
+    var parts = [];
+    var entry = self;
+    while (entry != null) {
+      parts.push(entry.name);
+      entry = entry.parent;
+    }
+    parts.reverse();
+    return parts;
+  };
+
+  AssistHdfsEntry.prototype.toggleOpen = function () {
+    var self = this;
+    self.open(!self.open());
+  };
+
+  return AssistHdfsEntry;
+}));

+ 68 - 17
desktop/core/src/desktop/static/desktop/js/assist/assistHelper.js

@@ -23,7 +23,10 @@
 }(this, function (ko) {
 }(this, function (ko) {
 
 
   var TIME_TO_LIVE_IN_MILLIS = 86400000; // 1 day
   var TIME_TO_LIVE_IN_MILLIS = 86400000; // 1 day
-  var API_PREFIX = "/notebook/api/autocomplete/";
+  var AUTOCOMPLETE_API_PREFIX = "/notebook/api/autocomplete/";
+  var HDFS_API_PREFIX = "/filebrowser/view=";
+  var HDFS_PARAMETERS = "?pagesize=100&format=json";
+
 
 
   /**
   /**
    * @param {Object} i18n
    * @param {Object} i18n
@@ -53,6 +56,39 @@
     return sourceType + "_" + self.user;
     return sourceType + "_" + self.user;
   };
   };
 
 
+  /**
+   * @param {string[]} pathParts
+   * @param {function} successCallback
+   * @param {function} errorCallback
+   * @param {Object} [editor] - Ace editor
+   */
+  AssistHelper.prototype.fetchHdfsPath = function (pathParts, successCallback, errorCallback, editor) {
+    var self = this;
+    var url = HDFS_API_PREFIX + "/" + pathParts.join("/") + HDFS_PARAMETERS;
+
+    var fetchFunction = function (successCallback) {
+      $.ajax({
+        dataType: "json",
+        url: url,
+        success: function (data) {
+          if (!data.error) {
+            successCallback(data);
+          } else {
+            errorCallback();
+          }
+        }
+      }).fail(function () {
+        errorCallback();
+      }).always(function () {
+        if (editor) {
+          editor.hideSpinner();
+        }
+      });
+    };
+
+    self.fetchCached('hdfs', url, fetchFunction, successCallback, editor);
+  };
+
   /**
   /**
    *
    *
    * @param {Object} options
    * @param {Object} options
@@ -67,7 +103,7 @@
     if (options.clearAll) {
     if (options.clearAll) {
       $.totalStorage("hue.assist." + self.getTotalStorageUserPrefix(options.sourceType), {});
       $.totalStorage("hue.assist." + self.getTotalStorageUserPrefix(options.sourceType), {});
     } else {
     } else {
-      var url = API_PREFIX;
+      var url = AUTOCOMPLETE_API_PREFIX;
       if (options.databaseName) {
       if (options.databaseName) {
         url += options.databaseName;
         url += options.databaseName;
       }
       }
@@ -91,7 +127,7 @@
   AssistHelper.prototype.loadDatabases = function (options) {
   AssistHelper.prototype.loadDatabases = function (options) {
     var self = this;
     var self = this;
 
 
-    self.fetchAssistData(options.sourceType, API_PREFIX, function(data) {
+    self.fetchAssistData(options.sourceType, AUTOCOMPLETE_API_PREFIX, function(data) {
       var databases = data.databases || [];
       var databases = data.databases || [];
       // Blacklist of system databases
       // Blacklist of system databases
       self.lastKnownDatabases = $.grep(databases, function(database) {
       self.lastKnownDatabases = $.grep(databases, function(database) {
@@ -271,7 +307,7 @@
    */
    */
   AssistHelper.prototype.fetchTables = function (options) {
   AssistHelper.prototype.fetchTables = function (options) {
     var self = this;
     var self = this;
-    self.fetchAssistData(options.sourceType, API_PREFIX + options.databaseName, options.successCallback, options.errorCallback, options.editor);
+    self.fetchAssistData(options.sourceType, AUTOCOMPLETE_API_PREFIX + options.databaseName, options.successCallback, options.errorCallback, options.editor);
   };
   };
 
 
   /**
   /**
@@ -287,7 +323,7 @@
   AssistHelper.prototype.fetchFields = function (options) {
   AssistHelper.prototype.fetchFields = function (options) {
     var self = this;
     var self = this;
     var fieldPart = options.fields.length > 0 ? "/" + options.fields.join("/") : "";
     var fieldPart = options.fields.length > 0 ? "/" + options.fields.join("/") : "";
-    self.fetchAssistData(options.sourceType, API_PREFIX + options.databaseName + "/" + options.tableName + fieldPart, options.successCallback, options.errorCallback, options.editor);
+    self.fetchAssistData(options.sourceType, AUTOCOMPLETE_API_PREFIX + options.databaseName + "/" + options.tableName + fieldPart, options.successCallback, options.errorCallback, options.editor);
   };
   };
 
 
   /**
   /**
@@ -299,7 +335,7 @@
    */
    */
   AssistHelper.prototype.fetchPanelData = function (options) {
   AssistHelper.prototype.fetchPanelData = function (options) {
     var self = this;
     var self = this;
-    self.fetchAssistData(options.sourceType, API_PREFIX + options.hierarchy.join("/"), options.successCallback, options.errorCallback);
+    self.fetchAssistData(options.sourceType, AUTOCOMPLETE_API_PREFIX + options.hierarchy.join("/"), options.successCallback, options.errorCallback);
   };
   };
 
 
   /**
   /**
@@ -311,13 +347,11 @@
    */
    */
   AssistHelper.prototype.fetchAssistData = function (sourceType, url, successCallback, errorCallback, editor) {
   AssistHelper.prototype.fetchAssistData = function (sourceType, url, successCallback, errorCallback, editor) {
     var self = this;
     var self = this;
-    if (!sourceType) { return };
-    var cachedData = $.totalStorage("hue.assist." + self.getTotalStorageUserPrefix(sourceType)) || {};
+    if (!sourceType) {
+      return
+    }
 
 
-    if (typeof cachedData[url] == "undefined" || self.hasExpired(cachedData[url].timestamp)) {
-      if (editor) {
-        editor.showSpinner();
-      }
+    var fetchFunction = function (successCallback) {
       $.post(url, {
       $.post(url, {
         notebook: {},
         notebook: {},
         snippet: ko.mapping.toJSON({
         snippet: ko.mapping.toJSON({
@@ -325,11 +359,6 @@
         })
         })
       }, function (data) {
       }, function (data) {
         if (data.status == 0) {
         if (data.status == 0) {
-          cachedData[url] = {
-            timestamp: (new Date()).getTime(),
-            data: data
-          };
-          $.totalStorage("hue.assist." + self.getTotalStorageUserPrefix(sourceType), cachedData);
           successCallback(data);
           successCallback(data);
         } else {
         } else {
           errorCallback(data);
           errorCallback(data);
@@ -339,6 +368,28 @@
           editor.hideSpinner();
           editor.hideSpinner();
         }
         }
       });
       });
+    };
+
+    self.fetchCached(sourceType, url, fetchFunction, successCallback, editor);
+  };
+
+  AssistHelper.prototype.fetchCached = function (sourceType, url, fetchFunction, successCallback, editor) {
+    var self = this;
+    var cachedData = $.totalStorage("hue.assist." + self.getTotalStorageUserPrefix(sourceType)) || {};
+
+    if (typeof cachedData[url] == "undefined" || self.hasExpired(cachedData[url].timestamp)) {
+      if (editor) {
+        editor.showSpinner();
+      }
+
+      fetchFunction(function (data) {
+        cachedData[url] = {
+          timestamp: (new Date()).getTime(),
+          data: data
+        };
+        $.totalStorage("hue.assist." + self.getTotalStorageUserPrefix(sourceType), cachedData);
+        successCallback(data);
+      });
     } else {
     } else {
       successCallback(cachedData[url].data);
       successCallback(cachedData[url].data);
     }
     }

+ 10 - 3
desktop/core/src/desktop/templates/assist.mako

@@ -321,11 +321,11 @@ from desktop.views import _ko
   <script type="text/javascript" charset="utf-8">
   <script type="text/javascript" charset="utf-8">
     (function (factory) {
     (function (factory) {
       if(typeof require === "function") {
       if(typeof require === "function") {
-        define('assistPanel', ['knockout', 'desktop/js/assist/assistDbSource', 'desktop/js/assist/assistHelper', 'tableStats'], factory);
+        define('assistPanel', ['knockout', 'desktop/js/assist/assistDbSource', 'desktop/js/assist/assistHdfsEntry', 'desktop/js/assist/assistHelper', 'tableStats'], factory);
       } else {
       } else {
-        factory(ko, AssistDbSource, AssistHelper);
+        factory(ko, AssistDbSource, AssistHdfsEntry, AssistHelper);
       }
       }
-    }(function (ko, AssistDbSource, AssistHelper) {
+    }(function (ko, AssistDbSource, AssistHdfsEntry, AssistHelper) {
 
 
       /**
       /**
        * @param {Object} params
        * @param {Object} params
@@ -361,6 +361,13 @@ from desktop.views import _ko
           self.sources.push(sourceIndex[sourceType.type]);
           self.sources.push(sourceIndex[sourceType.type]);
         });
         });
 
 
+        self.hdfsRoot = new AssistHdfsEntry({
+          name: '',
+          parent: null,
+          assistHelper: assistHelper
+        });
+        self.hdfsRoot.open(true);
+
         self.selectedSource = ko.observable(null);
         self.selectedSource = ko.observable(null);
 
 
         var setDatabaseWhenLoaded = function (databaseName) {
         var setDatabaseWhenLoaded = function (databaseName) {