generateParsers.js 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. /* eslint-disable no-restricted-syntax */
  17. const fs = require('fs');
  18. const exec = require('child_process').exec;
  19. const LICENSE =
  20. '// Licensed to Cloudera, Inc. under one\n' +
  21. '// or more contributor license agreements. See the NOTICE file\n' +
  22. '// distributed with this work for additional information\n' +
  23. '// regarding copyright ownership. Cloudera, Inc. licenses this file\n' +
  24. '// to you under the Apache License, Version 2.0 (the\n' +
  25. '// "License"); you may not use this file except in compliance\n' +
  26. '// with the License. You may obtain a copy of the License at\n' +
  27. '//\n' +
  28. '// http://www.apache.org/licenses/LICENSE-2.0\n' +
  29. '//\n' +
  30. '// Unless required by applicable law or agreed to in writing, software\n' +
  31. '// distributed under the License is distributed on an "AS IS" BASIS,\n' +
  32. '// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' +
  33. '// See the License for the specific language governing permissions and\n' +
  34. '// limitations under the License.\n';
  35. const SQL_STATEMENTS_PARSER_JSDOC =
  36. '/**\n' +
  37. ' * @param {string} input\n' +
  38. ' *\n' +
  39. ' * @return {SqlStatementsParserResult}\n' +
  40. ' */\n';
  41. const JISON_FOLDER = 'desktop/core/src/desktop/js/parse/jison/';
  42. const SQL_PARSER_REPOSITORY_PATH = 'desktop/core/src/desktop/js/parse/sql/sqlParserRepository.js';
  43. const SYNTAX_PARSER_IMPORT_TEMPLATE =
  44. ' KEY: () => import(/* webpackChunkName: "KEY-parser" */ \'parse/sql/KEY/KEYSyntaxParser\')';
  45. const AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE =
  46. ' KEY: () => import(/* webpackChunkName: "KEY-parser" */ \'parse/sql/KEY/KEYAutocompleteParser\')';
  47. const parserDefinitions = {
  48. globalSearchParser: {
  49. sources: ['globalSearchParser.jison'],
  50. target: 'globalSearchParser.jison',
  51. outputFolder: 'desktop/core/src/desktop/js/parse/',
  52. afterParse: contents =>
  53. new Promise(resolve => {
  54. resolve(
  55. LICENSE +
  56. contents.replace(
  57. 'var globalSearchParser = ',
  58. "import SqlParseSupport from 'parse/sqlParseSupport';\n\nvar globalSearchParser = "
  59. ) +
  60. '\nexport default globalSearchParser;\n'
  61. );
  62. })
  63. },
  64. solrFormulaParser: {
  65. sources: ['solrFormulaParser.jison'],
  66. target: 'solrFormulaParser.jison',
  67. outputFolder: 'desktop/core/src/desktop/js/parse/',
  68. afterParse: contents =>
  69. new Promise(resolve => {
  70. resolve(LICENSE + contents + 'export default solrFormulaParser;\n');
  71. })
  72. },
  73. solrQueryParser: {
  74. sources: ['solrQueryParser.jison'],
  75. target: 'solrQueryParser.jison',
  76. outputFolder: 'desktop/core/src/desktop/js/parse/',
  77. afterParse: contents =>
  78. new Promise(resolve => {
  79. resolve(LICENSE + contents + 'export default solrQueryParser;\n');
  80. })
  81. },
  82. sqlStatementsParser: {
  83. sources: ['sqlStatementsParser.jison'],
  84. target: 'sqlStatementsParser.jison',
  85. outputFolder: 'desktop/core/src/desktop/js/parse/',
  86. afterParse: contents =>
  87. new Promise(resolve => {
  88. resolve(
  89. LICENSE +
  90. contents.replace(
  91. 'parse: function parse',
  92. SQL_STATEMENTS_PARSER_JSDOC + 'parse: function parse'
  93. ) +
  94. 'export default sqlStatementsParser;\n'
  95. );
  96. })
  97. }
  98. };
  99. const mkdir = path =>
  100. new Promise((resolve, reject) => {
  101. if (fs.existsSync(path)) {
  102. resolve();
  103. } else {
  104. fs.mkdir(path, err => {
  105. if (err) {
  106. reject(err);
  107. }
  108. resolve();
  109. });
  110. }
  111. });
  112. const readFile = path =>
  113. new Promise((resolve, reject) => {
  114. fs.readFile(path, (err, buf) => {
  115. if (err) {
  116. reject(err);
  117. }
  118. resolve(buf ? buf.toString() : '');
  119. });
  120. });
  121. const writeFile = (path, contents) =>
  122. new Promise((resolve, reject) => {
  123. fs.writeFile(path, contents, err => {
  124. if (err) {
  125. reject();
  126. }
  127. resolve();
  128. });
  129. });
  130. const copyFile = (source, destination) =>
  131. new Promise((resolve, reject) => {
  132. readFile(source)
  133. .then(contents => {
  134. writeFile(destination, contents)
  135. .then(resolve)
  136. .catch(reject);
  137. })
  138. .catch(reject);
  139. });
  140. const deleteFile = path => {
  141. fs.unlinkSync(path);
  142. };
  143. const execCmd = cmd =>
  144. new Promise((resolve, reject) => {
  145. exec(cmd, (err, stdout, stderr) => {
  146. if (err) {
  147. reject('stderr:\n' + stderr + '\n\nstdout:\n' + stdout);
  148. }
  149. resolve(stdout);
  150. });
  151. });
  152. const generateParser = parserName =>
  153. new Promise((resolve, reject) => {
  154. const parserConfig = parserDefinitions[parserName];
  155. const concatPromise = new Promise((resolve, reject) => {
  156. if (parserConfig.sources.length > 1 && parserConfig.target) {
  157. console.log('Concatenating files...');
  158. const promises = parserConfig.sources.map(fileName => readFile(JISON_FOLDER + fileName));
  159. Promise.all(promises)
  160. .then(contents => {
  161. writeFile(JISON_FOLDER + parserConfig.target, contents.join('')).then(() => {
  162. resolve(JISON_FOLDER + parserConfig.target);
  163. });
  164. })
  165. .catch(reject);
  166. } else if (parserConfig.sources.length === 1) {
  167. resolve(JISON_FOLDER + parserConfig.sources[0]);
  168. } else {
  169. reject('No jison source specified');
  170. }
  171. });
  172. concatPromise
  173. .then(targetPath => {
  174. let jisonCommand = 'jison ' + targetPath;
  175. if (parserConfig.lexer) {
  176. jisonCommand += ' ' + JISON_FOLDER + parserConfig.lexer;
  177. }
  178. jisonCommand += ' -m js';
  179. console.log('Generating parser...');
  180. execCmd(jisonCommand)
  181. .then(stdout => {
  182. if (/\S/.test(stdout)) {
  183. console.log('got output for: ' + jisonCommand);
  184. console.log(stdout);
  185. }
  186. if (parserConfig.sources.length > 1) {
  187. deleteFile(targetPath); // Remove concatenated file
  188. }
  189. console.log('Adjusting JS...');
  190. const generatedJsFileName = parserConfig.target
  191. .replace('.jison', '.js')
  192. .replace(/^.*\/([^/]+)$/, '$1');
  193. readFile(generatedJsFileName)
  194. .then(contents => {
  195. parserConfig
  196. .afterParse(contents)
  197. .then(finalContents => {
  198. writeFile(parserConfig.outputFolder + generatedJsFileName, finalContents)
  199. .then(() => {
  200. deleteFile(generatedJsFileName);
  201. resolve();
  202. })
  203. .catch(reject);
  204. })
  205. .catch(reject);
  206. })
  207. .catch(reject);
  208. })
  209. .catch(reject);
  210. })
  211. .catch(reject);
  212. });
  213. let parsersToGenerate = [];
  214. const invalid = [];
  215. let all = false;
  216. const listDir = folder =>
  217. new Promise(resolve => {
  218. fs.readdir(folder, (err, files) => {
  219. resolve(files);
  220. });
  221. });
  222. const findParser = (fileIndex, folder, sharedFiles, autocomplete) => {
  223. const prefix = autocomplete ? 'autocomplete' : 'syntax';
  224. if (fileIndex[prefix + '_header.jison'] && fileIndex[prefix + '_footer.jison']) {
  225. const parserName = folder + (autocomplete ? 'AutocompleteParser' : 'SyntaxParser');
  226. const parserDefinition = {
  227. sources: ['sql/' + folder + '/' + prefix + '_header.jison'].concat(sharedFiles),
  228. lexer: 'sql/' + folder + '/sql.jisonlex',
  229. target: 'sql/' + folder + '/' + parserName + '.jison',
  230. sqlParser: autocomplete ? 'AUTOCOMPLETE' : 'SYNTAX',
  231. outputFolder: 'desktop/core/src/desktop/js/parse/sql/' + folder + '/',
  232. afterParse: contents =>
  233. new Promise(resolve => {
  234. resolve(
  235. LICENSE +
  236. contents
  237. .replace(
  238. 'var ' + parserName + ' = ',
  239. "import SqlParseSupport from 'parse/sql/" +
  240. folder +
  241. "/sqlParseSupport';\n\nvar " +
  242. parserName +
  243. ' = '
  244. )
  245. .replace(
  246. 'loc: yyloc,',
  247. "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),"
  248. ) +
  249. '\nexport default ' +
  250. parserName +
  251. ';\n'
  252. );
  253. })
  254. };
  255. parserDefinition.sources.push('sql/' + folder + '/' + prefix + '_footer.jison');
  256. parserDefinitions[parserName] = parserDefinition;
  257. } else {
  258. console.log(
  259. "Warn: Could not find '" +
  260. prefix +
  261. "_header.jison' or '" +
  262. prefix +
  263. "_footer.jison' in " +
  264. JISON_FOLDER +
  265. 'sql/' +
  266. folder +
  267. '/'
  268. );
  269. }
  270. };
  271. const identifySqlParsers = () =>
  272. new Promise(resolve => {
  273. listDir(JISON_FOLDER + 'sql').then(files => {
  274. const promises = [];
  275. files.forEach(folder => {
  276. promises.push(
  277. listDir(JISON_FOLDER + 'sql/' + folder).then(jisonFiles => {
  278. const fileIndex = {};
  279. jisonFiles.forEach(jisonFile => {
  280. fileIndex[jisonFile] = true;
  281. });
  282. const sharedFiles = jisonFiles
  283. .filter(jisonFile => jisonFile.indexOf('sql_') !== -1)
  284. .map(jisonFile => 'sql/' + folder + '/' + jisonFile);
  285. if (fileIndex['sql.jisonlex']) {
  286. findParser(fileIndex, folder, sharedFiles, true);
  287. findParser(
  288. fileIndex,
  289. folder,
  290. sharedFiles.filter(path => path.indexOf('_error.jison') === -1),
  291. false
  292. );
  293. } else {
  294. console.log(
  295. "Warn: Could not find 'sql.jisonlex' in " + JISON_FOLDER + 'sql/' + folder + '/'
  296. );
  297. }
  298. })
  299. );
  300. });
  301. Promise.all(promises).then(resolve);
  302. });
  303. });
  304. const prepareForNewParser = () =>
  305. new Promise((resolve, reject) => {
  306. if (process.argv.length === 3 && process.argv[0] === '-new') {
  307. process.argv.shift();
  308. const source = process.argv.shift();
  309. const target = process.argv.shift();
  310. console.log("Generating new parser '" + target + "' based on '" + source + "'...");
  311. process.argv.push(target);
  312. if (
  313. !Object.keys(parserDefinitions).some(key => {
  314. if (key.indexOf(source) === 0) {
  315. mkdir(JISON_FOLDER + 'sql/' + target)
  316. .then(() => {
  317. listDir(JISON_FOLDER + 'sql/' + source).then(files => {
  318. const copyPromises = [];
  319. files.forEach(file => {
  320. copyPromises.push(
  321. copyFile(
  322. JISON_FOLDER + 'sql/' + source + '/' + file,
  323. JISON_FOLDER + 'sql/' + target + '/' + file
  324. )
  325. );
  326. });
  327. Promise.all(copyPromises).then(() => {
  328. const autocompleteSources = ['sql/' + target + '/autocomplete_header.jison'];
  329. const syntaxSources = ['sql/' + target + '/syntax_header.jison'];
  330. const lexer = 'sql/' + target + '/sql.jisonlex';
  331. files.forEach(file => {
  332. if (file.indexOf('sql_') === 0) {
  333. autocompleteSources.push('sql/' + target + '/' + file);
  334. syntaxSources.push('sql/' + target + '/' + file);
  335. }
  336. });
  337. autocompleteSources.push('sql/' + target + '/autocomplete_footer.jison');
  338. syntaxSources.push('sql/' + target + '/syntax_footer.jison');
  339. mkdir('desktop/core/src/desktop/js/parse/sql/' + target).then(() => {
  340. readFile(
  341. 'desktop/core/src/desktop/js/parse/sql/' + source + '/sqlParseSupport.js'
  342. ).then(parseSupportContents => {
  343. parseSupportContents = parseSupportContents.replace(
  344. /parser\.yy\.activeDialect = '[^']+';'/g,
  345. "parser.yy.activeDialect = '" + target + "';"
  346. );
  347. writeFile(
  348. 'desktop/core/src/desktop/js/parse/sql/' + target + '/sqlParseSupport.js',
  349. parseSupportContents
  350. ).then(() => {
  351. parserDefinitions[target + 'AutocompleteParser'] = {
  352. sources: autocompleteSources,
  353. lexer: lexer,
  354. target: 'sql/' + target + '/' + target + 'AutocompleteParser.jison',
  355. sqlParser: 'AUTOCOMPLETE',
  356. outputFolder: 'desktop/core/src/desktop/js/parse/sql/' + target + '/',
  357. afterParse: contents =>
  358. new Promise(resolveAfterParse => {
  359. resolveAfterParse(
  360. LICENSE +
  361. contents
  362. .replace(
  363. 'var ' + target + 'AutocompleteParser = ',
  364. "import SqlParseSupport from 'parse/sql/" +
  365. target +
  366. "/sqlParseSupport';\n\nvar " +
  367. target +
  368. 'AutocompleteParser = '
  369. )
  370. .replace(
  371. 'loc: yyloc,',
  372. "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),"
  373. ) +
  374. '\nexport default ' +
  375. target +
  376. 'AutocompleteParser;\n'
  377. );
  378. })
  379. };
  380. parserDefinitions[target + 'SyntaxParser'] = {
  381. sources: syntaxSources,
  382. lexer: lexer,
  383. target: 'sql/' + target + '/' + target + 'SyntaxParser.jison',
  384. sqlParser: 'SYNTAX',
  385. outputFolder: 'desktop/core/src/desktop/js/parse/sql/' + target + '/',
  386. afterParse: contents =>
  387. new Promise(resolveAfterParse => {
  388. resolveAfterParse(
  389. LICENSE +
  390. contents
  391. .replace(
  392. 'var ' + target + 'SyntaxParser = ',
  393. "import SqlParseSupport from 'parse/sql/" +
  394. target +
  395. "/sqlParseSupport';\n\nvar " +
  396. target +
  397. 'SyntaxParser = '
  398. )
  399. .replace(
  400. 'loc: yyloc,',
  401. "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),"
  402. ) +
  403. '\nexport default ' +
  404. target +
  405. 'SyntaxParser;\n'
  406. );
  407. })
  408. };
  409. console.log(parserDefinitions);
  410. resolve();
  411. });
  412. });
  413. });
  414. });
  415. });
  416. })
  417. .catch(err => {
  418. console.log(err);
  419. });
  420. return true;
  421. }
  422. })
  423. ) {
  424. reject("No existing parser found for '" + source + "'");
  425. }
  426. } else {
  427. resolve();
  428. }
  429. });
  430. identifySqlParsers().then(() => {
  431. process.argv.shift();
  432. process.argv.shift();
  433. prepareForNewParser().then(() => {
  434. process.argv.forEach(arg => {
  435. if (arg === 'all') {
  436. all = true;
  437. } else if (parserDefinitions[arg]) {
  438. parsersToGenerate.push(arg);
  439. } else {
  440. let prefixFound = false;
  441. Object.keys(parserDefinitions).forEach(key => {
  442. if (key.indexOf(arg) === 0) {
  443. prefixFound = true;
  444. parsersToGenerate.push(key);
  445. }
  446. });
  447. if (!prefixFound) {
  448. invalid.push(arg);
  449. }
  450. }
  451. });
  452. if (all) {
  453. parsersToGenerate = Object.keys(parserDefinitions);
  454. }
  455. if (invalid.length) {
  456. console.log("No parser config found for: '" + invalid.join("', '") + "'");
  457. console.log(
  458. '\nPossible options are:\n ' +
  459. ['all'].concat(Object.keys(parserDefinitions)).join('\n ') +
  460. '\n'
  461. );
  462. return;
  463. }
  464. const parserCount = parsersToGenerate.length;
  465. let idx = 0;
  466. const generateRecursive = () => {
  467. idx++;
  468. if (parsersToGenerate.length) {
  469. const parserName = parsersToGenerate.pop();
  470. if (parserCount > 1) {
  471. console.log("Generating '" + parserName + "' (" + idx + '/' + parserCount + ')...');
  472. } else {
  473. console.log("Generating '" + parserName + "'...");
  474. }
  475. generateParser(parserName)
  476. .then(generateRecursive)
  477. .catch(error => {
  478. console.log(error);
  479. console.log('FAIL!');
  480. });
  481. } else {
  482. const autocompParsers = [];
  483. const syntaxParsers = [];
  484. console.log('Updating sqlParserRepository.js...');
  485. Object.keys(parserDefinitions).forEach(key => {
  486. if (parserDefinitions[key].sqlParser === 'AUTOCOMPLETE') {
  487. autocompParsers.push(
  488. AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE.replace(
  489. /KEY/g,
  490. key.replace('AutocompleteParser', '')
  491. )
  492. );
  493. } else if (parserDefinitions[key].sqlParser === 'SYNTAX') {
  494. syntaxParsers.push(
  495. SYNTAX_PARSER_IMPORT_TEMPLATE.replace(/KEY/g, key.replace('SyntaxParser', ''))
  496. );
  497. }
  498. });
  499. readFile(SQL_PARSER_REPOSITORY_PATH).then(contents => {
  500. contents = contents.replace(
  501. /const SYNTAX_MODULES = [^}]+}/,
  502. 'const SYNTAX_MODULES = {\n' + syntaxParsers.sort().join(',\n') + '\n}'
  503. );
  504. contents = contents.replace(
  505. /const AUTOCOMPLETE_MODULES = [^}]+}/,
  506. 'const AUTOCOMPLETE_MODULES = {\n' + autocompParsers.sort().join(',\n') + '\n}'
  507. );
  508. writeFile(SQL_PARSER_REPOSITORY_PATH, contents).then(() => {
  509. console.log('Done!\n');
  510. });
  511. });
  512. }
  513. };
  514. generateRecursive();
  515. });
  516. });
  517. /* eslint-enable no-restricted-syntax */