Prechádzať zdrojové kódy

HUE-2451 [dashboard] Initial generation setup of Solr query parser

Johan Ahlen 8 rokov pred
rodič
commit
520cf37263

+ 7 - 2
Gruntfile.js

@@ -63,7 +63,7 @@ module.exports = function(grunt) {
           'apps/security/src/security/static/security/css/security.css': 'apps/security/src/security/static/security/less/security.less',
           'apps/sqoop/src/sqoop/static/sqoop/css/sqoop.css': 'apps/sqoop/src/sqoop/static/sqoop/less/sqoop.less',
           'desktop/libs/indexer/src/indexer/static/indexer/css/indexes.css': 'desktop/libs/indexer/src/indexer/static/indexer/less/indexes.less',
-          'desktop/libs/indexer/src/indexer/static/indexer/css/importer.css': 'desktop/libs/indexer/src/indexer/static/indexer/less/importer.less',
+          'desktop/libs/indexer/src/indexer/static/indexer/css/importer.css': 'desktop/libs/indexer/src/indexer/static/indexer/less/importer.less'
         }
       }
     },
@@ -94,7 +94,7 @@ module.exports = function(grunt) {
           'apps/security/src/security/static/security/less/*.less',
           'apps/security/src/security/static/security/less/**/*.less',
           'apps/sqoop/src/sqoop/static/sqoop/less/*.less',
-          'apps/sqoop/src/sqoop/static/sqoop/less/**/*.less',
+          'apps/sqoop/src/sqoop/static/sqoop/less/**/*.less'
         ],
         tasks: ['less']
       }
@@ -124,6 +124,11 @@ module.exports = function(grunt) {
         files: {
           'desktop/core/src/desktop/static/desktop/js/autocomplete/jison/solrFormulaParser.js': ['desktop/core/src/desktop/static/desktop/js/autocomplete/jison/solrFormulaParser.js']
         }
+      },
+      solrQueryParser: {
+        files: {
+          'desktop/core/src/desktop/static/desktop/js/autocomplete/jison/solrQueryParser.js': ['desktop/core/src/desktop/static/desktop/js/autocomplete/jison/solrQueryParser.js']
+        }
       }
     }
   });

+ 8 - 0
Makefile

@@ -240,6 +240,14 @@ ace:
 global-search-parser:
 	@cd tools/jison && ./hue-global-search.sh
 
+.PHONY: solr-all-parsers
+solr-all-parsers:
+	@cd tools/jison && ./hue-solr-query.sh && ./hue-solr-formula.sh
+
+.PHONY: solr-query-parser
+solr-query-parser:
+	@cd tools/jison && ./hue-solr-query.sh
+
 .PHONY: solr-formula-parser
 solr-formula-parser:
 	@cd tools/jison && ./hue-solr-formula.sh

+ 178 - 0
desktop/core/src/desktop/static/desktop/js/autocomplete/jison/solrQueryParser.jison

@@ -0,0 +1,178 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+%lex
+%options case-insensitive
+%%
+
+\s                                         { /* skip whitespace */ }
+'--'.*                                     { /* skip comments */ }
+[/][*][^*]*[*]+([^/*][^*]*[*]+)*[/]        { /* skip comments */ }
+
+'('                                        { return '('; }
+')'                                        { return ')'; }
+
+'\u2020'                                   { parser.yy.cursorFound = yylloc; return 'CURSOR'; }
+
+[0-9]+(?:[,.][0-9]+)?                      { return 'NUMBER'; }
+
+[^\u2020()]+                               { parser.addFieldLocation(yylloc, yytext); return 'IDENTIFIER'; }
+
+<<EOF>>                                    { return 'EOF'; }
+
+/lex
+
+%start SolrQueryAutocomplete
+
+%%
+
+SolrQueryAutocomplete
+ : SolrQuery 'EOF'
+ | SolrQuery_EDIT 'EOF'
+   {
+     return $1;
+   }
+ | 'CURSOR' 'EOF'
+   {
+     return { suggestFields: true }
+   }
+ ;
+
+SolrQuery
+ : NonParenthesizedSolrQuery
+ | '(' NonParenthesizedSolrQuery ')'
+ ;
+
+SolrQuery
+ : NonParenthesizedSolrQuery_EDIT
+ | '(' NonParenthesizedSolrQuery_EDIT RightParenthesisOrError  --> $2
+ ;
+
+NonParenthesizedSolrQuery
+ : 'NUMBER'
+ | 'IDENTIFIER'
+ ;
+
+NonParenthesizedSolrQuery_EDIT
+ : 'NUMBER' 'CURSOR'                                                --> { suggestOperators: true }
+ | 'IDENTIFIER' 'CURSOR'                                            --> { suggestOperators: true }
+ | 'CURSOR' 'NUMBER'                                                --> { suggestFields: true }
+ | 'CURSOR' 'IDENTIFIER'                                            --> { suggestFields: true }
+ ;
+
+RightParenthesisOrError
+ : ')'
+ | error
+ ;
+
+%%
+
+parser.yy.parseError = function () { return false; }
+
+parser.addFieldLocation = function (location, name) {
+  parser.yy.locations.push({ type: 'field', name: name, location: adjustLocationForCursor(location) });
+}
+
+parser.identifyPartials = function (beforeCursor, afterCursor) {
+  var beforeMatch = beforeCursor.match(/[^()-*+/,\s]*$/);
+  var afterMatch = afterCursor.match(/^[^()-*+/,\s]*/);
+  return { left: beforeMatch ? beforeMatch[0].length : 0, right: afterMatch ? afterMatch[0].length : 0 };
+};
+
+var adjustLocationForCursor = function (location) {
+  // columns are 0-based and lines not, so add 1 to cols
+  var newLocation = {
+    first_line: location.first_line,
+    last_line: location.last_line,
+    first_column: location.first_column + 1,
+    last_column: location.last_column + 1
+  };
+  if (parser.yy.cursorFound) {
+    if (parser.yy.cursorFound.first_line === newLocation.first_line && parser.yy.cursorFound.last_column <= newLocation.first_column) {
+      var additionalSpace = parser.yy.partialLengths.left + parser.yy.partialLengths.right;
+      additionalSpace -= parser.yy.partialCursor ? 1 : 3; // For some reason the normal cursor eats 3 positions.
+      newLocation.first_column = newLocation.first_column + additionalSpace;
+      newLocation.last_column = newLocation.last_column + additionalSpace;
+    }
+  }
+  return newLocation;
+};
+
+parser.parseSolrQuery = function (query, debug) {
+  parser.yy.cursorFound = false;
+  parser.yy.locations = [];
+  query = query.replace(/\r\n|\n\r/gm, '\n');
+
+  var result;
+  try {
+    result = parser.parse(query);
+  } catch (err) {
+    if (debug) {
+      console.log(beforeCursor + '\u2020' + afterCursor);
+      console.log(err);
+      console.error(err.stack);
+    }
+  }
+  return result || false;
+}
+
+parser.autocompleteSolrQuery = function (beforeCursor, afterCursor, debug) {
+  parser.yy.cursorFound = false;
+  parser.yy.locations = [];
+
+  beforeCursor = beforeCursor.replace(/\r\n|\n\r/gm, '\n');
+  afterCursor = afterCursor.replace(/\r\n|\n\r/gm, '\n');
+
+  parser.yy.partialLengths = parser.identifyPartials(beforeCursor, afterCursor);
+
+  if (parser.yy.partialLengths.left > 0) {
+    beforeCursor = beforeCursor.substring(0, beforeCursor.length - parser.yy.partialLengths.left);
+  }
+
+  if (parser.yy.partialLengths.right > 0) {
+    afterCursor = afterCursor.substring(parser.yy.partialLengths.right);
+  }
+
+  var result;
+  try {
+    result = parser.parse(beforeCursor + '\u2020' + afterCursor);
+  } catch (err) {
+    // Workaround for too many missing parentheses (it's the only error we handle in the parser)
+    if (err && err.toString().indexOf('Parsing halted while starting to recover from another error') !== -1) {
+      var leftCount = (beforeCursor.match(/\(/g) || []).length;
+      var rightCount = (beforeCursor.match(/\)/g) || []).length;
+      var parenthesisPad = '';
+      while (rightCount < leftCount) {
+        parenthesisPad += ')';
+        rightCount++;
+      }
+      try {
+        result = parser.parse(beforeCursor + '\u2020' + parenthesisPad);
+      } catch (err) {
+        return {}
+      }
+    } else {
+      if (debug) {
+        console.log(beforeCursor + '\u2020' + afterCursor);
+        console.log(err);
+        console.error(err.stack);
+      }
+      return {}
+    }
+  }
+  result.locations = parser.yy.locations;
+  return result;
+};

Rozdielové dáta súboru neboli zobrazené, pretože súbor je príliš veľký
+ 15 - 0
desktop/core/src/desktop/static/desktop/js/autocomplete/solrQueryParser.js


+ 32 - 0
desktop/core/src/desktop/static/desktop/spec/autocomplete/solrQueryParserSpec.js

@@ -0,0 +1,32 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+(function () {
+  describe('solrQueryParser.js', function () {
+    var testAutocomplete = function (beforeCursor, afterCursor, expectedResult) {
+      var result = solrQueryParser.autocompleteSolrQuery(beforeCursor, afterCursor, true);
+      if (!expectedResult.locations) {
+        delete result.locations;
+      }
+      expect(result).toEqual(expectedResult);
+    };
+
+    it('should suggest fields for "|"', function () {
+      testAutocomplete('', '', {
+        suggestFields: true
+      });
+    });
+  });
+})();

+ 1 - 0
desktop/core/src/desktop/templates/common_header.mako

@@ -177,6 +177,7 @@ if USE_NEW_EDITOR.get():
   <script src="${ static('desktop/js/autocomplete/sqlParseSupport.js') }"></script>
   <script src="${ static('desktop/js/autocomplete/sqlStatementsParser.js') }"></script>
   <script src="${ static('desktop/js/autocomplete/sqlAutocompleteParser.js') }"></script>
+  <script src="${ static('desktop/js/autocomplete/solrQueryParser.js') }"></script>
   <script src="${ static('desktop/js/autocomplete/solrFormulaParser.js') }"></script>
   <script src="${ static('desktop/js/autocomplete/globalSearchParser.js') }"></script>
 

+ 1 - 0
desktop/core/src/desktop/templates/hue.mako

@@ -479,6 +479,7 @@ ${ commonshare() | n,unicode }
 <script src="${ static('desktop/js/autocomplete/sqlParseSupport.js') }"></script>
 <script src="${ static('desktop/js/autocomplete/sqlAutocompleteParser.js') }"></script>
 <script src="${ static('desktop/js/autocomplete/globalSearchParser.js') }"></script>
+<script src="${ static('desktop/js/autocomplete/solrQueryParser.js') }"></script>
 <script src="${ static('desktop/js/autocomplete/solrFormulaParser.js') }"></script>
 <script src="${ static('desktop/js/sqlAutocompleter.js') }"></script>
 <script src="${ static('desktop/js/sqlAutocompleter2.js') }"></script>

+ 3 - 0
desktop/core/src/desktop/templates/jasmineRunner.html

@@ -137,6 +137,9 @@
   <script type="text/javascript" src="../static/desktop/js/autocomplete/solrFormulaParser.js"></script>
   <script type="text/javascript" src="../static/desktop/spec/autocomplete/solrFormulaParserSpec.js"></script>
 
+  <script type="text/javascript" src="../static/desktop/js/autocomplete/solrQueryParser.js"></script>
+  <script type="text/javascript" src="../static/desktop/spec/autocomplete/solrQueryParserSpec.js"></script>
+
   <script type="text/javascript" charset="utf-8">
     if (/PhantomJS/.test(window.navigator.userAgent)) {
       console.log("Running Jasmine tests with JUnitXmlReporter and TerminalReporter");

+ 31 - 0
tools/jison/hue-solr-query.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "Make sure you install jison first (npm install jison -g)"
+echo ""
+echo "Generating parser..."
+
+pushd ../../desktop/core/src/desktop/static/desktop/js/autocomplete/jison
+
+echo "Creating SOLR Query parser..."
+jison solrQueryParser.jison
+grunt uglify:solrQueryParser
+cat license.txt solrQueryParser.js > ../solrQueryParser.js
+rm solrQueryParser.js
+
+popd
+echo "Done!"

Niektoré súbory nie sú zobrazené, pretože je v týchto rozdielových dátach zmenené mnoho súborov