Переглянути джерело

HUE-4031 [editor] Sort the error suggestions based on similarity

Johan Ahlen 8 роки тому
батько
коміт
b4ae7ffdc9

+ 114 - 0
desktop/core/src/desktop/static/desktop/js/autocomplete/spec/sqlParseSupportSpec.js

@@ -0,0 +1,114 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+(function () {
+  describe('sqlParseSupport.js', function () {
+
+    var expectDistance = function (strA, strB, distance, ignoreCase) {
+      var lr = SqlParseSupport.stringDistance(strA, strB, ignoreCase);
+      var rl = SqlParseSupport.stringDistance(strB, strA, ignoreCase);
+      expect(lr).toEqual(rl);
+      expect(lr).toEqual(distance);
+    };
+
+    it('should calculate the distance between "" and "" correctly', function () {
+      expectDistance('', '', 0, true);
+    });
+
+    it('should calculate the distance between "abc" and "" correctly', function () {
+      expectDistance('abc', '', 3, true);
+    });
+
+    it('should calculate the distance between "a" and "b" correctly', function () {
+      expectDistance('a', 'b', 1, true);
+    });
+
+    it('should calculate the distance between "abc" and "abc" correctly', function () {
+      expectDistance('abc', 'abc', 0, true);
+    });
+
+    it('should calculate the distance between "abcd" and "abc" correctly', function () {
+      expectDistance('abcd', 'abc', 1, true);
+    });
+
+    it('should calculate the distance between "abd" and "abc" correctly', function () {
+      expectDistance('abd', 'abc', 1, true);
+    });
+
+    it('should calculate the distance between "ca" and "abc" correctly', function () {
+      expectDistance('ca', 'abc', 3, true);
+    });
+
+    it('should calculate the distance between "abC" and "abc" whe not ignoring case correctly', function () {
+      expectDistance('abC', 'abc', 1, false);
+    });
+
+    it('should calculate the distance between "abC" and "abc" when ignoring case correctly', function () {
+      expectDistance('abC', 'abc', 0, true);
+    });
+
+    it('should calculate the distance between "abe" and "abc" correctly', function () {
+      expectDistance('abe', 'abc', 1, true);
+    });
+
+    it('should calculate the distance between "ace" and "abc" correctly', function () {
+      expectDistance('ace', 'abc', 2, true);
+    });
+
+    it('should calculate the distance between "12345" and "23451" correctly', function () {
+      expectDistance('12345', '23451', 2, true);
+    });
+
+    it('should calculate the distance between "abcde" and "12345" correctly', function () {
+      expectDistance('abcde', '12345', 5, true);
+    });
+
+    it('should calculate the distance between "12345" and "abcdefgh" correctly', function () {
+      expectDistance('12345', 'abcdefgh', 8, true);
+    });
+
+    it('should calculate the distance between "abc1def" and "abcdef" correctly', function () {
+      expectDistance('abc1def', 'abcdef', 1, true);
+    });
+
+    it('should calculate the distance between "bacdef" and "abcdef" correctly', function () {
+      expectDistance('bacdef', 'abcdef', 2, true);
+    });
+
+    xit ('should be quick', function () {
+      var strA = 'abcdefgh012345678ijklmnop012345678';
+      var strB = 'ijklmnop012345678abcdefgh012345678';
+      var start, end;
+      var durations = new Array(10000 - 1000);
+      for (var i = 0; i < 10000; i++) {
+        if (i > 1000) {
+          start = performance.now();
+        }
+        SqlParseSupport.stringDistance(strA, strB, true);
+        if (i > 1000) {
+          end = performance.now();
+          durations.push(end-start);
+        }
+      }
+      var sum = 0;
+      durations.forEach(function (duration) {
+        sum += duration;
+      });
+      console.log('it took ' + sum / durations.length + ' ms on average.');
+      // ~ 0.037 ms on average
+      expect(true).toBeTruthy();
+    })
+  });
+})();

+ 3 - 3
desktop/core/src/desktop/static/desktop/js/autocomplete/spec/sqlSyntaxParserSpec.js

@@ -57,14 +57,14 @@
     it('should suggest expected words for "SLELECT "', function() {
       var result = sqlSyntaxParser.parseSyntax('SLELECT ', '');
       expect(result).toBeTruthy();
-      expect(result.expected).toEqual(['ALTER', 'CREATE', 'DROP', 'FROM', 'INSERT', 'SELECT', 'SET', 'SHOW', 'TRUNCATE', 'UPDATE', 'USE', 'WITH']);
+      expect(result.expected).toEqual(['SELECT', 'SET', 'INSERT', 'ALTER', 'CREATE', 'SHOW', 'USE', 'WITH', 'FROM', 'DROP', 'TRUNCATE', 'UPDATE']);
     });
 
     describe('Hive specific', function () {
       it('should suggest expected words for "SLELECT "', function() {
         var result = sqlSyntaxParser.parseSyntax('SLELECT ', '', 'hive');
         expect(result).toBeTruthy();
-        expect(result.expected).toEqual(['ALTER', 'ANALYZE', 'CREATE', 'CREATE', 'DELETE', 'DESCRIBE', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'INSERT', 'LOAD', 'MSCK', 'RELOAD', 'REVOKE', 'SELECT', 'SET', 'SHOW', 'SHOW', 'TRUNCATE', 'UPDATE', 'USE', 'USE', 'WITH']);
+        expect(result.expected).toEqual(['SELECT', 'SET', 'DELETE', 'RELOAD', 'INSERT', 'ALTER', 'INSERT', 'CREATE', 'EXPORT', 'GRANT', 'SHOW', 'LOAD', 'IMPORT', 'EXPLAIN', 'CREATE', 'REVOKE', 'MSCK', 'ANALYZE', 'SHOW', 'USE', 'USE', 'TRUNCATE', 'DROP', 'UPDATE', 'WITH', 'FROM', 'DESCRIBE']);
       });
     });
 
@@ -72,7 +72,7 @@
       it('should suggest expected words for "SLELECT "', function() {
         var result = sqlSyntaxParser.parseSyntax('SLELECT ', '', 'impala');
         expect(result).toBeTruthy();
-        expect(result.expected).toEqual(['ALTER', 'COMPUTE', 'CREATE', 'CREATE', 'DESCRIBE', 'DROP', 'EXPLAIN', 'FROM', 'GRANT', 'INSERT', 'INSERT', 'INVALIDATE', 'LOAD', 'REFRESH', 'REVOKE', 'SELECT', 'SET', 'SHOW', 'TRUNCATE', 'UPDATE', 'USE', 'WITH']);
+        expect(result.expected).toEqual(['SELECT', 'SET', 'INSERT', 'ALTER', 'INSERT', 'GRANT', 'CREATE', 'CREATE', 'REVOKE', 'EXPLAIN', 'SHOW', 'USE', 'REFRESH', 'LOAD', 'COMPUTE', 'TRUNCATE', 'UPDATE', 'WITH', 'FROM', 'DROP', 'INVALIDATE', 'DESCRIBE']);
       });
     })
 

+ 66 - 2
desktop/core/src/desktop/static/desktop/js/autocomplete/sqlParseSupport.js

@@ -16,6 +16,66 @@
 
 var SqlParseSupport = (function () {
 
+  /**
+   * Calculates the Optimal String Alignment distance between two strings. Returns 0 when the strings are equal and the
+   * distance when not, distances is less than or equal to the length of the longest string.
+   *
+   * @param strA
+   * @param strB
+   * @param [ignoreCase]
+   * @returns {number} The similarity
+   */
+  var stringDistance = function (strA, strB, ignoreCase) {
+    if (ignoreCase) {
+      strA = strA.toLowerCase();
+      strB = strB.toLowerCase();
+    }
+
+    // TODO: Consider other algorithms for performance
+    var strALength = strA.length;
+    var strBLength = strB.length;
+    if (strALength === 0) {
+      return strBLength;
+    }
+    if (strBLength === 0) {
+      return strALength;
+    }
+
+    var distances = new Array(strALength);
+
+    var cost, deletion, insertion, substitution, transposition;
+    for (var i = 0; i <= strALength; i++) {
+      distances[i] = new Array(strBLength);
+      distances[i][0] = i;
+      for (var j = 1; j <= strBLength; j++) {
+        if (!i){
+          distances[0][j] = j;
+        } else {
+          cost = strA[i-1] === strB[j-1] ? 0 : 1;
+          deletion = distances[i - 1][j] + 1;
+          insertion = distances[i][j - 1] + 1;
+          substitution = distances[i - 1][j - 1] + cost;
+          if (deletion <= insertion && deletion <= substitution) {
+            distances[i][j] = deletion;
+          } else if (insertion <= deletion && insertion <= substitution) {
+            distances[i][j] = insertion;
+          } else {
+            distances[i][j] = substitution;
+          }
+
+          if (i > 1 && j > 1 && strA[i] === strB[j - 1] && strA[i - 1] === strB[j]) {
+            transposition = distances[i - 2][j - 2] + cost;
+            if (transposition < distances[i][j]) {
+              distances[i][j] = transposition;
+            }
+          }
+        }
+      }
+    }
+
+    return distances[strALength][strBLength];
+  };
+
   var equalIgnoreCase = function (a, b) {
     return a && b && a.toLowerCase() === b.toLowerCase();
   };
@@ -1542,7 +1602,10 @@ var SqlParseSupport = (function () {
             }
           }
         });
-        cleanExpected.sort();
+        cleanExpected.sort(function (a, b) {
+          // TODO: Possible performance boost when initializing the distances algorithm given parser.yy.error.text
+          return stringDistance(parser.yy.error.text, a, true) - stringDistance(parser.yy.error.text, b, true);
+        });
         parser.yy.error.expected = cleanExpected;
         return parser.yy.error;
       }
@@ -1552,6 +1615,7 @@ var SqlParseSupport = (function () {
 
   return {
     initSqlParser: initSqlParser,
-    initSyntaxParser: initSyntaxParser
+    initSyntaxParser: initSyntaxParser,
+    stringDistance: stringDistance
   };
 })();

+ 2 - 0
desktop/core/src/desktop/templates/jasmineRunner.html

@@ -103,6 +103,8 @@
   <script type="text/javascript" src="../static/desktop/spec/hdfsAutocompleterSpec.js"></script>
 
   <script type="text/javascript" src="../static/desktop/js/autocomplete/sqlParseSupport.js"></script>
+  <script type="text/javascript" src="../static/desktop/js/autocomplete/spec/sqlParseSupportSpec.js"></script>
+
   <script type="text/javascript" src="../static/desktop/js/autocomplete/sqlAutocompleteParser.js"></script>
   <script type="text/javascript" src="../static/desktop/js/autocomplete/spec/sqlSpec.js"></script>