Browse Source

HUE-6803 [editor] Let the statement parser return the first non-comment token of the statement

Johan Ahlen 8 years ago
parent
commit
8c640d5

+ 24 - 3
desktop/core/src/desktop/static/desktop/js/autocomplete/jison/sqlStatementsParser.jison

@@ -45,7 +45,15 @@
 <backTick><<EOF>>                                                     { this.popState(); return 'EOF'; }
 <backTick>'`'                                                         { this.popState(); return 'PART_OF_STATEMENT'; }
 
-[^"\/;'`-]+                                                           { return 'PART_OF_STATEMENT'; }
+[^"\/;'`-]+                                                           {
+                                                                        if (!parser.yy.firstToken) {
+                                                                          var firstWordMatch = yytext.match(/[a-zA-Z_]+/);
+                                                                          if (firstWordMatch) {
+                                                                            parser.yy.firstToken = firstWordMatch[0];
+                                                                          }
+                                                                        };
+                                                                        return 'PART_OF_STATEMENT';
+                                                                      }
 [-][^;-]                                                              { return 'PART_OF_STATEMENT'; }
 [/][^;*]                                                              { return 'PART_OF_STATEMENT'; }
 
@@ -97,11 +105,24 @@ SqlStatementsParser
  ;
 
 Statements
- : StatementParts                                                  -> [{ type: 'statement', statement: $1, location: @1 }]
+ : StatementParts
+   {
+     if (parser.yy.firstToken) {
+       $$ = [{ type: 'statement', statement: $1, location: @1, firstToken: parser.yy.firstToken }];
+       parser.yy.firstToken = null;
+     } else {
+       $$ = [{ type: 'statement', statement: $1, location: @1 }];
+     }
+   }
  | Statements OneOrMoreSeparators StatementParts
    {
      parser.handleTrailingStatements($1, $2);
-     $1.push({ type: 'statement', statement: $3, location: @3 });
+     if (parser.yy.firstToken) {
+       $1.push({ type: 'statement', statement: $3, location: @3, firstToken: parser.yy.firstToken });
+       parser.yy.firstToken = null;
+     } else {
+       $1.push({ type: 'statement', statement: $3, location: @3 });
+     }
    }
  ;
 

+ 23 - 3
desktop/core/src/desktop/static/desktop/js/autocomplete/sqlStatementsParser.js

@@ -138,12 +138,24 @@ case 6:
    
 break;
 case 7:
-this.$ = [{ type: 'statement', statement: $$[$0], location: _$[$0] }];
+
+     if (parser.yy.firstToken) {
+       this.$ = [{ type: 'statement', statement: $$[$0], location: _$[$0], firstToken: parser.yy.firstToken }];
+       parser.yy.firstToken = null;
+     } else {
+       this.$ = [{ type: 'statement', statement: $$[$0], location: _$[$0] }];
+     }
+   
 break;
 case 8:
 
      parser.handleTrailingStatements($$[$0-2], $$[$0-1]);
-     $$[$0-2].push({ type: 'statement', statement: $$[$0], location: _$[$0] });
+     if (parser.yy.firstToken) {
+       $$[$0-2].push({ type: 'statement', statement: $$[$0], location: _$[$0], firstToken: parser.yy.firstToken });
+       parser.yy.firstToken = null;
+     } else {
+       $$[$0-2].push({ type: 'statement', statement: $$[$0], location: _$[$0] });
+     }
    
 break;
 case 10:
@@ -710,7 +722,15 @@ case 19: this.popState(); return 5;
 break;
 case 20: this.popState(); return 8; 
 break;
-case 21: return 8; 
+case 21:
+                                                                        if (!parser.yy.firstToken) {
+                                                                          var firstWordMatch = yy_.yytext.match(/[a-zA-Z_]+/);
+                                                                          if (firstWordMatch) {
+                                                                            parser.yy.firstToken = firstWordMatch[0];
+                                                                          }
+                                                                        };
+                                                                        return 8;
+                                                                      
 break;
 case 22: return 8; 
 break;

+ 67 - 26
desktop/core/src/desktop/static/desktop/spec/autocomplete/sqlStatementsParserSpec.js

@@ -23,6 +23,9 @@
         s += first ? '{\n' : ', {\n';
         s += '  statement: \'' + entry.statement.replace(/\n/g, '\\n') + '\',\n';
         s += '  location: { first_line: ' + entry.location.first_line + ', first_column: ' + entry.location.first_column + ', last_line: ' + entry.location.last_line + ', last_column: ' + entry.location.last_column + ' }'
+        if (entry.firstToken) {
+          s+= ',\n  firstToken: \'' + entry.firstToken + '\''
+        }
         s += '\n}';
         first = false;
       });
@@ -73,7 +76,8 @@
     it('should split "select * from bla" correctly', function () {
       testParser('select * from bla', [{
         statement: 'select * from bla',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 17 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 17 },
+        firstToken: 'select'
       }]);
     });
 
@@ -86,157 +90,194 @@
         location: { first_line: 1, first_column: 1, last_line: 1, last_column: 2 }
       }, {
         statement: 'select * from bla',
-        location: { first_line: 1, first_column: 2, last_line: 1, last_column: 19 }
+        location: { first_line: 1, first_column: 2, last_line: 1, last_column: 19 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select * from bla;" correctly', function () {
       testParser('select * from bla;', [{
         statement: 'select * from bla;',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select * from bla;select * from ble" correctly', function () {
       testParser('select * from bla;select * from ble', [{
         statement: 'select * from bla;',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 },
+        firstToken: 'select'
       }, {
         statement: 'select * from ble',
-        location: { first_line: 1, first_column: 18, last_line: 1, last_column: 35 }
+        location: { first_line: 1, first_column: 18, last_line: 1, last_column: 35 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select * from bla;;select * from ble" correctly', function () {
       testParser('select * from bla;;select * from ble', [{
         statement: 'select * from bla;',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 },
+        firstToken: 'select'
       }, {
         statement: ';',
         location: { first_line: 1, first_column: 18, last_line: 1, last_column: 19 }
       },{
         statement: 'select * from ble',
-        location: { first_line: 1, first_column: 19, last_line: 1, last_column: 36 }
+        location: { first_line: 1, first_column: 19, last_line: 1, last_column: 36 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select * from bla;\\n;select * from ble" correctly', function () {
       testParser('select * from bla;\n;select * from ble', [{
         statement: 'select * from bla;',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 18 },
+        firstToken: 'select'
       }, {
         statement: '\n;',
         location: { first_line: 1, first_column: 18, last_line: 2, last_column: 1 }
       },{
         statement: 'select * from ble',
-        location: { first_line: 2, first_column: 1, last_line: 2, last_column: 18 }
+        location: { first_line: 2, first_column: 1, last_line: 2, last_column: 18 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select * \\nfrom bla;\\r\\nselect * from ble;\\n" correctly', function () {
       testParser('select * \nfrom bla;\r\nselect * from ble;\n', [{
         statement: 'select * \nfrom bla;',
-        location: { first_line: 1, first_column: 0, last_line: 2, last_column: 9 }
+        location: { first_line: 1, first_column: 0, last_line: 2, last_column: 9 },
+        firstToken: 'select'
       }, {
         statement: '\r\nselect * from ble;',
-        location: { first_line: 2, first_column: 9, last_line: 3, last_column: 18 }
+        location: { first_line: 2, first_column: 9, last_line: 3, last_column: 18 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select * from bla where x = ";";" correctly', function () {
       testParser('select * from bla where x = ";";', [{
         statement: 'select * from bla where x = ";";',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 32 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 32 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select * from bla where x = \';\';\\n\\nSELECT bla FROM foo WHERE y = `;` AND true = false;" correctly', function () {
       testParser('select * from bla where x = \';\';\n\nSELECT bla FROM foo WHERE y = `;` AND true = false;', [{
         statement: 'select * from bla where x = \';\';',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 32 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 32 },
+        firstToken: 'select'
       }, {
         statement: '\n\nSELECT bla FROM foo WHERE y = `;` AND true = false;',
-        location: { first_line: 1, first_column: 32, last_line: 3, last_column: 51 }
+        location: { first_line: 1, first_column: 32, last_line: 3, last_column: 51 },
+        firstToken: 'SELECT'
       }]);
     });
 
     it('should split "select * from bla where x = "; AND boo = 1;\\n\\nUSE db" correctly', function () {
       testParser('select * from bla where x = "; AND boo = 1;\n\nUSE db', [{
         statement: 'select * from bla where x = "; AND boo = 1;\n\nUSE db',
-        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 6 }
+        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 6 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "--- Some comment with ; ; \\nselect * from bla where x = ";";" correctly', function () {
       testParser('--- Some comment with ; ; \nselect * from bla where x = ";";', [{
         statement: '--- Some comment with ; ; \nselect * from bla where x = ";";',
-        location: { first_line: 1, first_column: 0, last_line: 2, last_column: 32 }
+        location: { first_line: 1, first_column: 0, last_line: 2, last_column: 32 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select *\n-- bla\n from bla;" correctly', function () {
       testParser('select *\n-- bla\n from bla;', [{
         statement: 'select *\n-- bla\n from bla;',
-        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 10 }
+        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 10 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "select *\\n/* bla \\n\\n*/\\n from bla;" correctly', function () {
       testParser('select *\n/* bla \n\n*/\n from bla;', [{
         statement: 'select *\n/* bla \n\n*/\n from bla;',
-        location: { first_line: 1, first_column: 0, last_line: 5, last_column: 10 }
+        location: { first_line: 1, first_column: 0, last_line: 5, last_column: 10 },
+        firstToken: 'select'
       }]);
     });
 
     it('should split "SELECT\\n id -- some ID\\n FROM customers;" correctly', function () {
       testParser('SELECT\n id -- some ID\n FROM customers;', [{
         statement: 'SELECT\n id -- some ID\n FROM customers;',
-        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 16 }
+        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 16 },
+        firstToken: 'SELECT'
       }]);
     });
 
     it('should split "SELECT\n id -- some ID;\n FROM customers;" correctly', function () {
       testParser('SELECT\n id -- some ID;\n FROM customers;', [{
         statement: 'SELECT\n id -- some ID;\n FROM customers;',
-        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 16 }
+        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 16 },
+        firstToken: 'SELECT'
       }]);
     });
 
     it('should split "SELECT id\\n\\n /* from customers; */ FROM other;" correctly', function () {
       testParser('SELECT id\n\n /* from customers; */ FROM other;', [{
         statement: 'SELECT id\n\n /* from customers; */ FROM other;',
-        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 34 }
+        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 34 },
+        firstToken: 'SELECT'
       }]);
     });
 
     it('should split "SELECT `   " correctly', function () {
       testParser('SELECT `   ', [{
         statement: 'SELECT `   ',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 11 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 11 },
+        firstToken: 'SELECT'
       }]);
     });
 
     it('should split "SELECT "   " correctly', function () {
       testParser('SELECT "   ', [{
         statement: 'SELECT "   ',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 11 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 11 },
+        firstToken: 'SELECT'
       }]);
     });
 
     it('should split "SELECT \'   " correctly', function () {
       testParser('SELECT \'   ', [{
         statement: 'SELECT \'   ',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 11 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 11 },
+        firstToken: 'SELECT'
+      }]);
+    });
+
+    it('should split "-- Some comment\\n\\n    CREATE TABLE bla (id int); /* some \\nother comment*/ ALTER TABLE boo" correctly', function () {
+      testParser('-- Some comment\n\n    CREATE TABLE bla (id int); /* some \nother comment*/ ALTER TABLE boo', [{
+        statement: '-- Some comment\n\n    CREATE TABLE bla (id int);',
+        location: { first_line: 1, first_column: 0, last_line: 3, last_column: 30 },
+        firstToken: 'CREATE'
+      }, {
+        statement: ' /* some \nother comment*/ ALTER TABLE boo',
+        location: { first_line: 3, first_column: 30, last_line: 4, last_column: 31 },
+        firstToken: 'ALTER'
       }]);
     });
 
     it('should split "SELECT " \\" ;; ", \'"\', \' ;\' from bla; /* \\n\\n"" ; \\n; */ FROM other;" correctly', function () {
       testParser('USE `db;`;\r\nSELECT " \\" ;; ", \'"\', \' ;\' from bla; /* \n\n"" ; \n;  FROM other;*/', [{
         statement: 'USE `db;`;',
-        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 10 }
+        location: { first_line: 1, first_column: 0, last_line: 1, last_column: 10 },
+        firstToken: 'USE'
       }, {
         statement: '\r\nSELECT " \\" ;; ", \'"\', \' ;\' from bla;',
-        location: { first_line: 1, first_column: 10, last_line: 2, last_column: 37 }
+        location: { first_line: 1, first_column: 10, last_line: 2, last_column: 37 },
+        firstToken: 'SELECT'
       }, {
         statement: ' /* \n\n"" ; \n;  FROM other;*/',
         location: { first_line: 2, first_column: 37, last_line: 5, last_column: 16 }