Bläddra i källkod

[notebook] Add HDFS autocomplete

Johan Ahlen 10 år sedan
förälder
incheckning
930e70d

+ 1 - 0
apps/beeswax/src/beeswax/templates/execute.mako

@@ -787,6 +787,7 @@ ${ commonshare() | n,unicode }
 <script src="${ static('desktop/ext/js/knockout-mapping.min.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/ko.hue-bindings.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/sqlAutocompleter.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('desktop/js/hdfsAutocompleter.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/assistHelper.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/autocompleter.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('notebook/js/notebook.ko.js') }" type="text/javascript" charset="utf-8"></script>

+ 7 - 4
desktop/core/src/desktop/static/desktop/js/autocompleter.js

@@ -16,11 +16,14 @@
 
 (function (root, factory) {
   if(typeof define === "function" && define.amd) {
-    define(['desktop/js/sqlAutocompleter'], factory);
+    define([
+      'desktop/js/sqlAutocompleter',
+      'desktop/js/HdfsAutocompleter'
+    ], factory);
   } else {
-    root.Autocompleter = factory(SqlAutocompleter);
+    root.Autocompleter = factory(SqlAutocompleter, HdfsAutocompleter);
   }
-}(this, function (SqlAutocompleter) {
+}(this, function (SqlAutocompleter, HdfsAutocompleter) {
 
   /**
    * @param options {object}
@@ -37,7 +40,7 @@
           snippet: options.snippet
         })
       } else {
-        self.autocompleter = null;
+        self.autocompleter = new HdfsAutocompleter();
       }
     };
     self.snippet.type.subscribe(function () {

+ 110 - 0
desktop/core/src/desktop/static/desktop/js/hdfsAutocompleter.js

@@ -0,0 +1,110 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+(function (root, factory) {
+  if(typeof define === "function" && define.amd) {
+    define([], factory);
+  } else {
+    root.HdfsAutocompleter = factory();
+  }
+}(this, function () {
+
+  var BASE_PATH = "/filebrowser/view=";
+  var PARAMETERS = "?pagesize=100&format=json";
+
+  /**
+   * @param options {object}
+   *
+   * @constructor
+   */
+  function HdfsAutocompleter(options) {
+    var self = this;
+  }
+
+  HdfsAutocompleter.prototype.fetchAlternatives = function (pathParts, success, error, editor) {
+    var self = this;
+    if (editor) {
+      editor.showSpinner();
+    }
+
+    $.ajax({
+      dataType: "json",
+      url: BASE_PATH + "/" + pathParts.join("/") + PARAMETERS,
+      success: function (data) {
+        if (editor) {
+          editor.hideSpinner();
+        }
+        if (! data.error) {
+          success(self.extractFields(data, pathParts.length > 0));
+        } else {
+          error();
+        }
+      }
+    }).fail(function () {
+      if (editor) {
+        editor.hideSpinner();
+      }
+      error();
+    });
+  };
+
+  HdfsAutocompleter.prototype.extractFields = function (data, includeParent) {
+    var files = $.map(data.files, function (file) {
+      return {
+        name: file.name,
+        type: file.type
+      }
+    });
+
+    files.sort(function (a, b) {
+      return a.name.localeCompare(b.name);
+    });
+
+    var result = [];
+    files.forEach(function(field, idx) {
+      if (field.name !== '..' && field.name !== '.') {
+        result.push({
+          value: field.name,
+          score: 1000 - idx,
+          meta: field.type
+        });
+      }
+    });
+    return result;
+  };
+
+  HdfsAutocompleter.prototype.autocomplete = function (beforeCursor, afterCursor, callback, editor) {
+    var self = this;
+
+    var onFailure = function () {
+      callback([]);
+    };
+
+    if (beforeCursor.match(/["'](?:\/[^\/]*)+/)) {
+      var parts = beforeCursor.split('/');
+      // Drop the first " or '
+      parts.shift();
+      // Last one is either partial name or empty
+      parts.pop();
+
+      self.fetchAlternatives(parts, callback, onFailure, editor);
+    } else {
+      onFailure();
+    }
+  };
+
+  return HdfsAutocompleter;
+}));

+ 43 - 0
desktop/core/src/desktop/static/desktop/spec/autocompleterTestUtils.js

@@ -0,0 +1,43 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+define([
+], function() {
+  return {
+    autocompleteMatcher : {
+      toEqualAutocompleteValues : function() {
+        return {
+          compare: function(actualItems, expectedValues) {
+            var itemIndex = {};
+
+            if (actualItems.length !== expectedValues.length) {
+              return { pass: false };
+            }
+            $.each(actualItems, function(i, item) {
+              itemIndex[item.value] = true;
+            });
+
+            for (var i = 0; i < expectedValues.length; i++) {
+              if (! itemIndex[expectedValues[i]]) {
+                return { pass: false };
+              }
+            }
+            return { pass: true };
+          }
+        }
+      }
+    }
+  }
+});

+ 148 - 0
desktop/core/src/desktop/static/desktop/spec/hdfsAutocompleterSpec.js

@@ -0,0 +1,148 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+define([
+  'knockout',
+  'desktop/js/hdfsAutocompleter',
+  'desktop/spec/autocompleterTestUtils'
+], function(ko, HdfsAutocompleter, testUtils) {
+  describe("hdfsAutocompleter.js", function() {
+    var subject;
+
+    var ajaxHelper = {
+      responseForUrls: {}
+    };
+
+    beforeAll(function() {
+      jasmine.addMatchers(testUtils.autocompleteMatcher);
+      $.totalStorage = function(key, value) {
+        return null;
+      };
+
+      spyOn($, "ajax").and.callFake(function(options) {
+        var firstUrlPart = options.url.split("?")[0];
+        var response;
+        expect(ajaxHelper.responseForUrls[firstUrlPart]).toBeDefined("fake response for url " + firstUrlPart + " not found");
+        response = ajaxHelper.responseForUrls[firstUrlPart];
+        response.called = true;
+        response.status = 0;
+        options.success(response);
+        return({
+          fail: function() {
+            return {
+              always: $.noop
+            }
+          }
+        })
+      });
+    });
+
+    afterEach(function() {
+      $.each(ajaxHelper.responseForUrls, function(key, value) {
+        expect(value.called).toEqual(true, key + " was never called");
+      })
+    });
+
+    beforeEach(function() {
+      subject = new HdfsAutocompleter();
+      ajaxHelper.responseForUrls = {};
+    });
+
+    var createCallbackSpyForValues = function(values, name) {
+      return jasmine.createSpy(name ? name : 'callback', function (value) {
+        expect(value).toEqualAutocompleteValues(values)
+      }).and.callThrough();
+    };
+
+    var assertAutoComplete = function(testDefinition) {
+      ajaxHelper.responseForUrls = testDefinition.serverResponses;
+      var callback = createCallbackSpyForValues(testDefinition.expectedSuggestions);
+      subject.autocomplete(testDefinition.beforeCursor, testDefinition.afterCursor, callback);
+      expect(callback).toHaveBeenCalled();
+    };
+
+    it("should return empty suggestions for empty statement", function() {
+      assertAutoComplete({
+        serverResponses: {},
+        beforeCursor: "",
+        afterCursor: "",
+        expectedSuggestions: []
+      });
+    });
+
+    it("should return empty suggestions for bogus statements", function() {
+      assertAutoComplete({
+        serverResponses: {},
+        beforeCursor: "qwerqwer'asdf/",
+        afterCursor: "",
+        expectedSuggestions: []
+      });
+    });
+
+    it("should return suggestions for root with '", function() {
+      assertAutoComplete({
+        serverResponses: {
+          "/filebrowser/view=/" : {
+            files: [
+              { name: ".", type: "dir" },
+              { name: "..", type: "dir" },
+              { name: "var", type: "dir" },
+              { name: "tmp_file", type: "file" }
+            ]
+          }
+        },
+        beforeCursor: "'/",
+        afterCursor: "",
+        expectedSuggestions: ["tmp_file", "var"]
+      });
+    });
+
+    it("should return suggestions for root with \"", function() {
+      assertAutoComplete({
+        serverResponses: {
+          "/filebrowser/view=/" : {
+            files: [
+              { name: ".", type: "dir" },
+              { name: "..", type: "dir" },
+              { name: "var", type: "dir" },
+              { name: "tmp_file", type: "file" }
+            ]
+          }
+        },
+        beforeCursor: "\"/",
+        afterCursor: "",
+        expectedSuggestions: ["tmp_file", "var"]
+      });
+    });
+
+    it("should return suggestions for non-root", function() {
+      assertAutoComplete({
+        serverResponses: {
+          "/filebrowser/view=/foo/bar" : {
+            files: [
+              { name: ".", type: "dir" },
+              { name: "..", type: "dir" },
+              { name: "var", type: "dir" },
+              { name: "tmp_file", type: "file" }
+            ]
+          }
+        },
+        beforeCursor: "'/foo/bar/",
+        afterCursor: "",
+        expectedSuggestions: ["tmp_file", "var"]
+      });
+    });
+  });
+});

+ 4 - 27
desktop/core/src/desktop/static/desktop/spec/sqlAutocompleterSpec.js

@@ -16,8 +16,9 @@
 define([
   'knockout',
   'desktop/js/sqlAutocompleter',
-  'desktop/js/assistHelper'
-], function(ko, SqlAutocompleter, AssistHelper) {
+  'desktop/js/assistHelper',
+  'desktop/spec/autocompleterTestUtils'
+], function(ko, SqlAutocompleter, AssistHelper, testUtils) {
   describe("sqlAutocompleter.js", function() {
     var subject;
 
@@ -45,34 +46,10 @@ define([
     };
 
     beforeAll(function() {
+      jasmine.addMatchers(testUtils.autocompleteMatcher);
       $.totalStorage = function(key, value) {
         return null;
       };
-
-      jasmine.addMatchers({
-        toEqualAutocompleteValues : function() {
-          return {
-            compare: function(actualItems, expectedValues) {
-              var itemIndex = {};
-
-              if (actualItems.length !== expectedValues.length) {
-                return { pass: false };
-              }
-              $.each(actualItems, function(i, item) {
-                itemIndex[item.value] = true;
-              });
-
-              for (var i = 0; i < expectedValues.length; i++) {
-                if (! itemIndex[expectedValues[i]]) {
-                  return { pass: false };
-                }
-              }
-              return { pass: true };
-            }
-          }
-        }
-      });
-
       spyOn($, "ajax").and.callFake(function(options) {
         var firstUrlPart = options.url.split("?")[0];
         var response;

+ 3 - 2
desktop/core/src/desktop/templates/jasmine.mako

@@ -22,10 +22,11 @@
 
 <%block name="specs">
   <script type="text/javascript" charset="utf-8">
-  require(['jasmine-boot'], function () {
+  require(['jasmine-boot', 'jasmine'], function () {
     // Add specs below
     require([
-      'desktop/spec/sqlAutocompleterSpec'
+      'desktop/spec/sqlAutocompleterSpec',
+      'desktop/spec/hdfsAutocompleterSpec'
     ], function() {
       window.onload();
     });