瀏覽代碼

[beeswax] Support autocomplete for exploded lateral views

This adds autocomplete support for exploded and pos-exploded lateral views of maps and arrays.
Johan Ahlen 10 年之前
父節點
當前提交
383f4d0

+ 99 - 4
desktop/core/src/desktop/static/desktop/js/autocomplete.js

@@ -45,7 +45,7 @@ Autocompleter.prototype.getTableReferenceIndex = function (statement) {
   var fromMatch = statement.match(/\s*from\s*([^;]*).*$/i);
   if (fromMatch) {
     var tableRefsRaw = fromMatch[1];
-    upToMatch = tableRefsRaw.match(/\bON|LIMIT|WHERE|GROUP|SORT|ORDER BY\b/i);
+    var upToMatch = tableRefsRaw.match(/\bLATERAL|VIEW|EXPLODE|POSEXPLODE|ON|LIMIT|WHERE|GROUP|SORT|ORDER BY\b/i);
     if (upToMatch) {
       tableRefsRaw = $.trim(tableRefsRaw.substring(0, upToMatch.index));
     }
@@ -59,7 +59,76 @@ Autocompleter.prototype.getTableReferenceIndex = function (statement) {
   return result;
 };
 
-Autocompleter.prototype.extractFields = function (data, valuePrefix, includeStar) {
+Autocompleter.prototype.getViewReferenceIndex = function (statement) {
+  var result = {
+    allViewReferences: [],
+    index: {}
+  };
+
+  // Matches both arrays and maps "AS ref" or "AS (keyRef, valueRef)" and with
+  // or without view reference.
+  // group 1 = pos for posexplode or undefined
+  // group 2 = argument to table generating function
+  // group 3 = view reference or undefined
+  // group 4 = array item reference
+  //           array index reference (if posexplode)
+  //           map key reference (if group 5 is exists)
+  // group 5 = array value (if posexplode)
+  //           map value reference (if ! posexplode)
+  var lateralViewRegex = /.*LATERAL\s+VIEW\s+(pos)?explode\(([^\)]+)\)\s+(?:(\S+)\s+)?AS \(?([^ ,\)]*)(?:\s*,\s*([^ ,\)]*))?/gi;
+  var lateralViewMatch;
+
+  while (lateralViewMatch = lateralViewRegex.exec(statement)) {
+    var isMapRef = (!lateralViewMatch[1] && lateralViewMatch[5]) || false ;
+    var isPosexplode = lateralViewMatch[1] || false;
+
+    var pathToField = lateralViewMatch[2].split(".");
+
+    var viewRef = {
+      leadingPath: pathToField,
+      references: []
+    };
+
+    if (isMapRef) {
+      // TODO : use lateralViewMatch[4] for key ref once API supports map key lookup
+      result.index[lateralViewMatch[5]] = {
+        leadingPath: pathToField,
+        addition: 'value'
+      };
+      viewRef.references.push({ name: lateralViewMatch[4], type: 'key'});
+      viewRef.references.push({ name: lateralViewMatch[5], type: 'value'});
+    } else if (isPosexplode) {
+      // TODO : use lateralViewMatch[4] for array index ref once API supports array index lookup
+      // Currently we don't support array key refs
+      result.index[lateralViewMatch[5]] = {
+        leadingPath: pathToField,
+        addition: 'item'
+      };
+      viewRef.references.push({ name: lateralViewMatch[4], type: 'index'});
+      viewRef.references.push({ name: lateralViewMatch[5], type: 'item'});
+    } else {
+      // exploded array without position
+      result.index[lateralViewMatch[4]] = {
+        leadingPath: pathToField,
+        addition: 'item'
+      };
+      viewRef.references.push({ name: lateralViewMatch[4], type: 'item'});
+    }
+
+    result.allViewReferences = result.allViewReferences.concat(viewRef.references);
+
+    if(lateralViewMatch[3]) {
+      result.allViewReferences.push({ name: lateralViewMatch[3], type: 'view' });
+      result.index[lateralViewMatch[3]] = viewRef;
+    }
+  }
+
+  // TODO: Support view references from views. (expand result values using result, limit steps to prevent infinite refs)
+
+  return result;
+};
+
+Autocompleter.prototype.extractFields = function (data, valuePrefix, includeStar, references) {
   var fields = [];
   var result = [];
 
@@ -89,6 +158,10 @@ Autocompleter.prototype.extractFields = function (data, valuePrefix, includeStar
     });
   }
 
+  if (references) {
+    fields = fields.concat(references);
+  }
+
   fields.sort(function (a, b) {
     return a.name.localeCompare(b.name);
   });
@@ -141,7 +214,6 @@ Autocompleter.prototype.autocomplete = function(beforeCursor, afterCursor, callb
     beforeMatcher[beforeMatcher.length - 1] === "ON" ||
     beforeMatcher[beforeMatcher.length - 1] === "ORDER BY");
 
-
   if (tableNameAutoComplete || (selectBefore && !fromAfter)) {
     self.assistHelper.fetchTables(function(data) {
       var fromKeyword = "";
@@ -191,10 +263,15 @@ Autocompleter.prototype.autocomplete = function(beforeCursor, afterCursor, callb
       return;
     }
 
+    var viewReferences = self.getViewReferenceIndex(beforeCursor + afterCursor);
     var getFields = function (remainingParts, fields) {
       if (remainingParts.length == 0) {
         self.assistHelper.fetchFields(tableName, fields, function(data) {
-          callback(self.extractFields(data, "", !fieldTermBefore));
+          if (fields.length == 0) {
+            callback(self.extractFields(data, "", !fieldTermBefore, viewReferences.allViewReferences));
+          } else {
+            callback(self.extractFields(data, "", !fieldTermBefore));
+          }
         }, onFailure);
         return; // break recursion
       }
@@ -202,6 +279,20 @@ Autocompleter.prototype.autocomplete = function(beforeCursor, afterCursor, callb
 
       if (part != '' && part !== tableName) {
         if (self.currentMode === "hive") {
+          if (viewReferences.index[part]) {
+            if (viewReferences.index[part].references && remainingParts.length == 0) {
+              callback(self.extractFields([], "", true, viewReferences.index[part].references));
+              return;
+            }
+            if (fields.length == 0) {
+              fields.push(viewReferences.index[part].leadingPath);
+            }
+            if (viewReferences.index[part].addition) {
+              fields.push(viewReferences.index[part].addition);
+            }
+            getFields(remainingParts, fields);
+            return;
+          }
           var mapOrArrayMatch = part.match(/([^\[]*)\[[^\]]*\]$/i);
           if (mapOrArrayMatch !== null) {
             fields.push(mapOrArrayMatch[1]);
@@ -224,6 +315,10 @@ Autocompleter.prototype.autocomplete = function(beforeCursor, afterCursor, callb
       getFields(remainingParts, fields);
     };
 
+    parts = parts.filter(function(value) {
+      return value != '';
+    });
+
     getFields(parts, []);
   } else {
     onFailure();

+ 122 - 15
desktop/core/src/desktop/static/desktop/spec/autocompleteSpec.js

@@ -50,13 +50,23 @@ describe("autocomplete.js", function() {
         }
       }
     });
+
     spyOn($, "ajax").and.callFake(function(options) {
       var firstUrlPart = options.url.split("?")[0];
       expect(ajaxHelper.responseForUrls[firstUrlPart]).toBeDefined("fake response for url " + firstUrlPart + " not found");
-      options.success(ajaxHelper.responseForUrls[firstUrlPart]);
+      if (ajaxHelper.responseForUrls[firstUrlPart]) {
+        ajaxHelper.responseForUrls[firstUrlPart].called = true;
+        options.success(ajaxHelper.responseForUrls[firstUrlPart]);
+      }
     });
   });
 
+  afterEach(function() {
+    $.each(ajaxHelper.responseForUrls, function(key, value) {
+      expect(value.called).toEqual(true, key + " was never called");
+    })
+  });
+
   beforeEach(function() {
     var options = {
       assistHelper: new AssistHelper({
@@ -268,6 +278,115 @@ describe("autocomplete.js", function() {
         expectedSuggestions: ["fieldA", "fieldB"]
       });
     });
+
+    describe("lateral views", function() {
+      it("should suggest structs from exploded item references to arrays", function () {
+        assertAutoComplete({
+          serverResponses: {
+            "/testApp/api/autocomplete/testDb/testTable/testArray/item": {
+              fields: [
+                {"type": "string", "name": "fieldA"},
+                {"type": "string", "name": "fieldB"}
+              ],
+              type: "struct"
+            }
+          },
+          beforeCursor: "SELECT testItem.",
+          afterCursor: " FROM testTable LATERAL VIEW explode(testArray) explodedTable AS testItem",
+          expectedSuggestions: ["fieldA", "fieldB"]
+        });
+      });
+
+      it("should suggest structs from references to exploded arrays", function () {
+        assertAutoComplete({
+          serverResponses: {
+            "/testApp/api/autocomplete/testDb/testTable/testArray/item": {
+              fields: [
+                {"type": "string", "name": "fieldA"},
+                {"type": "string", "name": "fieldB"}
+              ],
+              type: "struct"
+            }
+          },
+          beforeCursor: "SELECT explodedTable.testItem.",
+          afterCursor: " FROM testTable LATERAL VIEW explode(testArray) explodedTable AS testItem",
+          expectedSuggestions: ["fieldA", "fieldB"]
+        });
+      });
+
+      it("should suggest posexploded references to arrays", function () {
+        assertAutoComplete({
+          serverResponses: {
+            "/testApp/api/autocomplete/testDb/testTable/testArray/item": {
+              fields: [
+                {"type": "string", "name": "fieldA"},
+                {"type": "string", "name": "fieldB"}
+              ],
+              type: "struct"
+            }
+          },
+          beforeCursor: "SELECT testValue.",
+          afterCursor: " FROM testTable LATERAL VIEW posexplode(testArray) explodedTable AS (testIndex, testValue)",
+          expectedSuggestions: ["fieldA", "fieldB"]
+        });
+      });
+
+      it("should suggest exploded references to map values", function () {
+        assertAutoComplete({
+          serverResponses: {
+            "/testApp/api/autocomplete/testDb/testTable/testMap/value": {
+              fields: [
+                {"type": "string", "name": "fieldA"},
+                {"type": "string", "name": "fieldB"}
+              ],
+              type: "struct"
+            }
+          },
+          beforeCursor: "SELECT testMapValue.",
+          afterCursor: " FROM testTable LATERAL VIEW explode(testMap) AS (testMapKey, testMapValue)",
+          expectedSuggestions: ["fieldA", "fieldB"]
+        });
+      });
+
+      it("should suggest exploded references to map values from view references", function () {
+        assertAutoComplete({
+          serverResponses: {
+            "/testApp/api/autocomplete/testDb/testTable/testMap/value": {
+              fields: [
+                {"type": "string", "name": "fieldA"},
+                {"type": "string", "name": "fieldB"}
+              ],
+              type: "struct"
+            }
+          },
+          beforeCursor: "SELECT explodedMap.testMapValue.",
+          afterCursor: " FROM testTable LATERAL VIEW explode(testMap) explodedMap AS (testMapKey, testMapValue)",
+          expectedSuggestions: ["fieldA", "fieldB"]
+        });
+      });
+
+      it("should suggest references to exploded references from view reference", function () {
+        assertAutoComplete({
+          serverResponses: {},
+          beforeCursor: "SELECT explodedMap.",
+          afterCursor: " FROM testTable LATERAL VIEW explode(testMap) explodedMap AS (testMapKey, testMapValue)",
+          expectedSuggestions: ["testMapKey", "testMapValue"]
+        });
+      });
+
+      it("should suggest references to exploded references", function () {
+        assertAutoComplete({
+          serverResponses: {
+            "/testApp/api/autocomplete/testDb/testTable" : {
+              columns: ["testTableColumn1", "testTableColumn2"]
+            }
+          },
+          beforeCursor: "SELECT ",
+          afterCursor: " FROM testTable LATERAL VIEW explode(testMap) explodedMap AS (testMapKey, testMapValue)",
+          expectedSuggestions: ["*", "explodedMap", "testTableColumn1", "testTableColumn2", "testMapKey", "testMapValue"]
+        });
+      });
+    });
   });
 
   describe("impala-specific stuff", function() {
@@ -286,9 +405,7 @@ describe("autocomplete.js", function() {
 
     it("should not suggest struct from map values with hive style syntax", function() {
       assertAutoComplete({
-        serverResponses: {
-          "/testApp/api/autocomplete/testDb/testTable/testMap[\"anyKey\"]" : {}
-        },
+        serverResponses: {},
         beforeCursor: "SELECT testMap[\"anyKey\"].",
         afterCursor: " FROM testTable",
         expectedSuggestions: []
@@ -342,9 +459,6 @@ describe("autocomplete.js", function() {
         serverResponses: {
           "/testApp/api/autocomplete/testDb/testTable1" : {
             columns: ["testTableColumn1", "testTableColumn2"]
-          },
-          "/testDb/testTable2" : {
-            columns: ["testTableColumn3", "testTableColumn4"]
           }
         },
         beforeCursor: "SELECT t1.testTableColumn1, t2.testTableColumn3 FROM testTable1 t1 JOIN testTable2 t2 ON t1.",
@@ -404,14 +518,7 @@ describe("autocomplete.js", function() {
 
     it("should suggest aliases", function() {
       assertAutoComplete({
-        serverResponses: {
-          "/testApp/api/autocomplete/testDb/testTableA" : {
-            columns: ["testTableColumn1", "testTableColumn2"]
-          },
-          "/testApp/api/autocomplete/testDb/testTableB" : {
-            columns: ["testTableColumn3", "testTableColumn4"]
-          }
-        },
+        serverResponses: {},
         beforeCursor: "SELECT ",
         afterCursor: " FROM testTableA tta, testTableB",
         expectedSuggestions: ["testTableB.", "tta."]