瀏覽代碼

HUE-8762 [frontend] Stabilize parser generation

By switching to node for generation the resulting parser should be the same when generated across different platforms. This also makes it easier to extend Hue with new parsers.
Johan Ahlen 6 年之前
父節點
當前提交
e434298be7

+ 8 - 8
Makefile

@@ -241,35 +241,35 @@ ace:
 # <<<< DEV ONLY
 .PHONY: global-search-parser
 global-search-parser:
-	@cd tools/jison && ./hue-global-search.sh
+	@node tools/jison/generateParsers.js globalSearchParser
 
 .PHONY: solr-all-parsers
 solr-all-parsers:
-	@cd tools/jison && ./hue-solr-query.sh && ./hue-solr-formula.sh
+	@node tools/jison/generateParsers.js solrQueryParser solrFormulaParser
 
 .PHONY: solr-query-parser
 solr-query-parser:
-	@cd tools/jison && ./hue-solr-query.sh
+	@node tools/jison/generateParsers.js solrQueryParser
 
 .PHONY: solr-formula-parser
 solr-formula-parser:
-	@cd tools/jison && ./hue-solr-formula.sh
+	@node tools/jison/generateParsers.js solrFormulaParser
 
 .PHONY: sql-all-parsers
 sql-all-parsers:
-	@cd tools/jison && ./hue-sql-autocomplete.sh && ./hue-sql-statement.sh && ./hue-sql-syntax.sh
+	@node tools/jison/generateParsers.js sqlAutocompleteParser sqlSyntaxParser sqlStatementsParser
 
 .PHONY: sql-autocomplete-parser
 sql-autocomplete-parser:
-	@cd tools/jison && ./hue-sql-autocomplete.sh
+	@node tools/jison/generateParsers.js sqlAutocompleteParser
 
 .PHONY: sql-statement-parser
 sql-statement-parser:
-	@cd tools/jison && ./hue-sql-statement.sh
+	@node tools/jison/generateParsers.js sqlStatementsParser
 
 .PHONY: sql-syntax-parser
 sql-syntax-parser:
-	@cd tools/jison && ./hue-sql-syntax.sh
+	@node tools/jison/generateParsers.js sqlSyntaxParser
 # END DEV ONLY >>>>
 
 ###################################

+ 0 - 15
desktop/core/src/desktop/js/parse/jison/license.txt

@@ -1,15 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.

+ 227 - 0
tools/jison/generateParsers.js

@@ -0,0 +1,227 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+let fs = require('fs');
+let exec = require('child_process').exec;
+
+const LICENSE = '// Licensed to Cloudera, Inc. under one\n' +
+  '// or more contributor license agreements.  See the NOTICE file\n' +
+  '// distributed with this work for additional information\n' +
+  '// regarding copyright ownership.  Cloudera, Inc. licenses this file\n' +
+  '// to you under the Apache License, Version 2.0 (the\n' +
+  '// "License"); you may not use this file except in compliance\n' +
+  '// with the License.  You may obtain a copy of the License at\n' +
+  '//\n' +
+  '//     http://www.apache.org/licenses/LICENSE-2.0\n' +
+  '//\n' +
+  '// Unless required by applicable law or agreed to in writing, software\n' +
+  '// distributed under the License is distributed on an "AS IS" BASIS,\n' +
+  '// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' +
+  '// See the License for the specific language governing permissions and\n' +
+  '// limitations under the License.\n';
+
+const JISON_FOLDER = 'desktop/core/src/desktop/js/parse/jison/';
+const TARGET_FOLDER = 'desktop/core/src/desktop/js/parse/';
+
+const PARSERS = {
+  globalSearchParser: {
+    sources: ['globalSearchParser.jison'],
+    target: 'globalSearchParser.jison',
+    afterParse: (contents) => new Promise(resolve => {
+      resolve(LICENSE +
+        contents.replace('var globalSearchParser = ', 'import SqlParseSupport from \'parse/sqlParseSupport\';\n\nvar globalSearchParser = ') +
+        '\nexport default globalSearchParser;\n');
+    })
+  },
+  solrFormulaParser: {
+    sources: ['solrFormulaParser.jison'],
+    target: 'solrFormulaParser.jison',
+    afterParse: (contents) => new Promise(resolve => {
+      resolve(LICENSE + contents + 'export default solrFormulaParser;\n');
+    })
+  },
+  solrQueryParser: {
+    sources: ['solrQueryParser.jison'],
+    target: 'solrQueryParser.jison',
+    afterParse: (contents) => new Promise(resolve => {
+      resolve(LICENSE + contents + 'export default solrQueryParser;\n');
+    })
+  },
+  sqlAutocompleteParser: {
+    sources: [
+      'autocomplete_header.jison', 'sql_main.jison', 'sql_valueExpression.jison', 'sql_error.jison', 'sql_alter.jison',
+      'sql_analyze.jison', 'sql_create.jison', 'sql_drop.jison', 'sql_grant.jison', 'sql_insert.jison', 'sql_load.jison',
+      'sql_set.jison', 'sql_show.jison', 'sql_update.jison', 'sql_use.jison', 'autocomplete_footer.jison'
+    ],
+    target: 'sqlAutocompleteParser.jison',
+    lexer: 'sql.jisonlex',
+    afterParse: (contents) => new Promise(resolve => {
+      resolve(LICENSE +
+        contents.replace('var sqlAutocompleteParser = ', 'import SqlParseSupport from \'parse/sqlParseSupport\';\n\nvar sqlAutocompleteParser = ') +
+        '\nexport default sqlAutocompleteParser;\n');
+    })
+  },
+  sqlStatementsParser: {
+    sources: ['sqlStatementsParser.jison'],
+    target: 'sqlStatementsParser.jison',
+    afterParse: (contents) => new Promise(resolve => {
+      resolve(LICENSE + contents + 'export default sqlStatementsParser;\n');
+    })
+  },
+  sqlSyntaxParser: {
+    sources: [
+      'syntax_header.jison', 'sql_main.jison', 'sql_valueExpression.jison', 'sql_alter.jison', 'sql_analyze.jison',
+      'sql_create.jison', 'sql_drop.jison', 'sql_grant.jison', 'sql_insert.jison', 'sql_load.jison', 'sql_set.jison',
+      'sql_show.jison', 'sql_update.jison', 'sql_use.jison', 'syntax_footer.jison'
+    ],
+    target: 'sqlSyntaxParser.jison',
+    lexer: 'sql.jisonlex',
+    afterParse: (contents) => new Promise(resolve => {
+      resolve(LICENSE +
+        contents.replace('var sqlSyntaxParser = ', 'import SqlParseSupport from \'parse/sqlParseSupport\';\n\nvar sqlSyntaxParser = ')
+          .replace('loc: yyloc,', 'loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(\'\'),') +
+        '\nexport default sqlSyntaxParser;\n');
+    })
+  },
+};
+
+const readFile = (path) => new Promise((resolve, reject) => {
+  fs.readFile(path, (err, buf) => {
+    if (err) {
+      reject();
+    }
+    resolve(buf.toString());
+  })
+});
+
+const writeFile = (path, contents) => new Promise((resolve, reject) => {
+  fs.writeFile(path, contents, function(err, data) {
+    if (err) {
+      reject();
+    }
+    resolve();
+  });
+});
+
+const deleteFile = (path) => {
+  fs.unlinkSync(path);
+};
+
+const execCmd = (cmd) => new Promise((resolve, reject) => {
+  exec(cmd, function(err, stdout, stderr) {
+    if (err) {
+      reject();
+    }
+    resolve();
+  });
+});
+
+const generateParser = parserName => new Promise((resolve, reject) => {
+  let parserConfig = PARSERS[parserName];
+
+  let concatPromise = new Promise((resolve, reject) => {
+    if (parserConfig.sources.length > 1 && parserConfig.target) {
+      console.log('Concatenating files...');
+      let promises = parserConfig.sources.map(fileName => readFile(JISON_FOLDER + fileName));
+
+      Promise.all(promises).then(contents => {
+        writeFile(JISON_FOLDER + parserConfig.target, contents).then(() => {
+          resolve(JISON_FOLDER + parserConfig.target)
+        })
+      }).catch(reject);
+    } else if (parserConfig.sources.length === 1) {
+      resolve(JISON_FOLDER + parserConfig.sources[0]);
+    } else {
+      reject('No jison source specified');
+    }
+  });
+
+
+  concatPromise.then((targetPath) => {
+    let jisonCommand = 'jison ' + targetPath;
+    if (parserConfig.lexer) {
+      jisonCommand += ' ' + JISON_FOLDER + parserConfig.lexer
+    }
+    jisonCommand += ' -m js';
+    console.log('Generating parser...');
+    execCmd(jisonCommand).then(() => {
+      if (parserConfig.sources.length > 1) {
+        deleteFile(targetPath); // Remove concatenated file
+      }
+      console.log('Adjusting JS...');
+      let generatedJsFileName = parserConfig.target.replace('.jison', '.js');
+      readFile(generatedJsFileName).then(contents => {
+        parserConfig.afterParse(contents).then(finalContents => {
+          writeFile(TARGET_FOLDER + generatedJsFileName, finalContents).then(() => {
+            deleteFile(generatedJsFileName);
+            console.log('Done!\n');
+            resolve();
+          }).catch(reject);
+        }).catch(reject);
+      }).catch(reject);
+    }).catch(reject);
+  }).catch(reject);
+});
+
+let parsersToGenerate = [];
+const invalid = [];
+
+let all = false;
+let appFound = false;
+process.argv.forEach(arg => {
+  if (appFound) {
+    if (arg === 'all') {
+      all = true;
+    } else if (PARSERS[arg]) {
+      parsersToGenerate.push(arg);
+    } else {
+      invalid.push(arg);
+    }
+  } else if (arg.indexOf('generateParsers.js') !== -1) {
+    appFound = true;
+  }
+});
+
+if (all) {
+  parsersToGenerate = Object.keys(PARSERS);
+}
+
+if (invalid.length) {
+  console.log('No parser config found for: \'' + invalid.join('\', \'') + '\'');
+  console.log('\nPossible options are:\n  ' + ['all'].concat(Object.keys(PARSERS)).join('\n  ') + '\n');
+  return;
+}
+
+parserCount = parsersToGenerate.length;
+let idx = 0;
+
+const generateRecursive = () => {
+  idx++;
+  if (parsersToGenerate.length) {
+    let parserName = parsersToGenerate.pop();
+    if (parserCount > 1) {
+      console.log('Generating \'' + parserName + '\' (' + idx + '/' + parserCount + ')...');
+    } else {
+      console.log('Generating \'' + parserName + '\'...');
+    }
+    generateParser(parserName).then(generateRecursive)
+  }
+};
+
+generateRecursive();
+
+
+

+ 0 - 35
tools/jison/hue-global-search.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Make sure you install jison first (npm install jison -g)"
-echo ""
-echo "Generating parser..."
-
-pushd ../../desktop/core/src/desktop/js/parse/jison
-
-echo "Creating Global Search parser..."
-jison globalSearchParser.jison -m js
-
-# Add ES6 style import/export
-sed -i '' $'s#var globalSearchParser = #import SqlParseSupport from \'parse/sqlParseSupport\';\\\n\\\nvar globalSearchParser = #' globalSearchParser.js
-echo 'export default globalSearchParser;' >> globalSearchParser.js
-
-cat license.txt globalSearchParser.js > ../globalSearchParser.js
-rm globalSearchParser.js
-
-popd
-echo "Done!"

+ 0 - 34
tools/jison/hue-solr-formula.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Make sure you install jison first (npm install jison -g)"
-echo ""
-echo "Generating parser..."
-
-pushd ../../desktop/core/src/desktop/js/parse/jison
-
-echo "Creating SOLR Formula parser..."
-jison solrFormulaParser.jison -m js
-
-# Add ES6 style export
-echo 'export default solrFormulaParser;' >> solrFormulaParser.js
-
-cat license.txt solrFormulaParser.js > ../solrFormulaParser.js
-rm solrFormulaParser.js
-
-popd
-echo "Done!"

+ 0 - 34
tools/jison/hue-solr-query.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Make sure you install jison first (npm install jison -g)"
-echo ""
-echo "Generating parser..."
-
-pushd ../../desktop/core/src/desktop/js/parse/jison
-
-echo "Creating SOLR Query parser..."
-jison solrQueryParser.jison -m js
-
-# Add ES6 style export
-echo 'export default solrQueryParser;' >> solrQueryParser.js
-
-cat license.txt solrQueryParser.js > ../solrQueryParser.js
-rm solrQueryParser.js
-
-popd
-echo "Done!"

+ 0 - 40
tools/jison/hue-sql-autocomplete.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Make sure you install jison first (npm install jison -g)"
-echo ""
-echo "Generating parser..."
-
-pushd ../../desktop/core/src/desktop/js/parse/jison
-
-# For quick version of just SELECT statements without value expressions
-# cat autocomplete_header.jison sql_main.jison autocomplete_footer.jison > sqlAutocompleteParser.jison
-cat autocomplete_header.jison sql_main.jison sql_valueExpression.jison sql_error.jison sql_alter.jison sql_analyze.jison sql_create.jison sql_drop.jison sql_grant.jison sql_insert.jison sql_load.jison sql_set.jison sql_show.jison sql_update.jison sql_use.jison autocomplete_footer.jison > sqlAutocompleteParser.jison
-
-echo "Creating SQL autocomplete parser..."
-jison sqlAutocompleteParser.jison sql.jisonlex -m js
-
-# Add ES6 style import/export
-sed -i '' $'s#var sqlAutocompleteParser = #import SqlParseSupport from \'parse/sqlParseSupport\';\\\n\\\nvar sqlAutocompleteParser = #' sqlAutocompleteParser.js
-echo 'export default sqlAutocompleteParser;' >> sqlAutocompleteParser.js
-
-cat license.txt sqlAutocompleteParser.js > ../sqlAutocompleteParser.js
-rm sqlAutocompleteParser.jison
-rm sqlAutocompleteParser.js
-
-popd
-echo "Done!"

+ 0 - 34
tools/jison/hue-sql-statement.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Make sure you install jison first (npm install jison -g)"
-echo ""
-echo "Generating parser..."
-
-pushd ../../desktop/core/src/desktop/js/parse/jison
-
-echo "Creating SQL statements parser..."
-jison sqlStatementsParser.jison -m js
-
-# Add ES6 style export
-echo 'export default sqlStatementsParser;' >> sqlStatementsParser.js
-
-cat license.txt sqlStatementsParser.js > ../sqlStatementsParser.js
-rm sqlStatementsParser.js
-
-popd
-echo "Done!"

+ 0 - 43
tools/jison/hue-sql-syntax.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Make sure you install jison first (npm install jison -g)"
-echo ""
-echo "Generating parser..."
-
-pushd ../../desktop/core/src/desktop/js/parse/jison
-
-# === Syntax parser ===
-cat syntax_header.jison sql_main.jison sql_valueExpression.jison  sql_alter.jison sql_analyze.jison sql_create.jison sql_drop.jison sql_grant.jison sql_insert.jison sql_load.jison sql_set.jison sql_show.jison sql_update.jison sql_use.jison syntax_footer.jison > sqlSyntaxParser.jison
-
-echo "Creating SQL syntax parser..."
-jison sqlSyntaxParser.jison sql.jisonlex -m js
-
-# Workaround for a parser bug where it reports the location of the previous token on error (pull-request submitted for jison)
-# We're also adding a ruleId to the parser error composed of the last two stack ID's and used for suppressing errors in the UI
-sed -i '' 's/loc: yyloc,/loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),/' sqlSyntaxParser.js
-
-# Add ES6 style import/export
-sed -i '' $'s#var sqlSyntaxParser = #import SqlParseSupport from \'parse/sqlParseSupport\';\\\n\\\nvar sqlSyntaxParser = #' sqlSyntaxParser.js
-echo 'export default sqlSyntaxParser;' >> sqlSyntaxParser.js
-
-cat license.txt sqlSyntaxParser.js > ../sqlSyntaxParser.js
-rm sqlSyntaxParser.jison
-rm sqlSyntaxParser.js
-
-popd
-echo "Done!"