generateParsers.js 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. const fs = require('fs');
  17. const exec = require('child_process').exec;
  18. const LICENSE =
  19. '// Licensed to Cloudera, Inc. under one\n' +
  20. '// or more contributor license agreements. See the NOTICE file\n' +
  21. '// distributed with this work for additional information\n' +
  22. '// regarding copyright ownership. Cloudera, Inc. licenses this file\n' +
  23. '// to you under the Apache License, Version 2.0 (the\n' +
  24. '// "License"); you may not use this file except in compliance\n' +
  25. '// with the License. You may obtain a copy of the License at\n' +
  26. '//\n' +
  27. '// http://www.apache.org/licenses/LICENSE-2.0\n' +
  28. '//\n' +
  29. '// Unless required by applicable law or agreed to in writing, software\n' +
  30. '// distributed under the License is distributed on an "AS IS" BASIS,\n' +
  31. '// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' +
  32. '// See the License for the specific language governing permissions and\n' +
  33. '// limitations under the License.\n';
  34. const SQL_STATEMENTS_PARSER_JSDOC =
  35. '/**\n' +
  36. ' * @param {string} input\n' +
  37. ' *\n' +
  38. ' * @return {SqlStatementsParserResult}\n' +
  39. ' */\n';
  40. const JISON_FOLDER = 'desktop/core/src/desktop/js/parse/jison/';
  41. const SQL_PARSER_REPOSITORY_PATH = 'desktop/core/src/desktop/js/parse/sql/sqlParserRepository.js';
  42. const SYNTAX_PARSER_IMPORT_TEMPLATE =
  43. ' KEY: () => import(/* webpackChunkName: "KEY-parser" */ \'parse/sql/KEY/KEYSyntaxParser\')';
  44. const AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE =
  45. ' KEY: () => import(/* webpackChunkName: "KEY-parser" */ \'parse/sql/KEY/KEYAutocompleteParser\')';
  46. const parserDefinitions = {
  47. globalSearchParser: {
  48. sources: ['globalSearchParser.jison'],
  49. target: 'globalSearchParser.jison',
  50. outputFolder: 'desktop/core/src/desktop/js/parse/',
  51. afterParse: contents =>
  52. new Promise(resolve => {
  53. resolve(
  54. LICENSE +
  55. contents.replace(
  56. 'var globalSearchParser = ',
  57. "import SqlParseSupport from 'parse/sqlParseSupport';\n\nvar globalSearchParser = "
  58. ) +
  59. '\nexport default globalSearchParser;\n'
  60. );
  61. })
  62. },
  63. solrFormulaParser: {
  64. sources: ['solrFormulaParser.jison'],
  65. target: 'solrFormulaParser.jison',
  66. outputFolder: 'desktop/core/src/desktop/js/parse/',
  67. afterParse: contents =>
  68. new Promise(resolve => {
  69. resolve(LICENSE + contents + 'export default solrFormulaParser;\n');
  70. })
  71. },
  72. solrQueryParser: {
  73. sources: ['solrQueryParser.jison'],
  74. target: 'solrQueryParser.jison',
  75. outputFolder: 'desktop/core/src/desktop/js/parse/',
  76. afterParse: contents =>
  77. new Promise(resolve => {
  78. resolve(LICENSE + contents + 'export default solrQueryParser;\n');
  79. })
  80. },
  81. sqlStatementsParser: {
  82. sources: ['sqlStatementsParser.jison'],
  83. target: 'sqlStatementsParser.jison',
  84. outputFolder: 'desktop/core/src/desktop/js/parse/',
  85. afterParse: contents =>
  86. new Promise(resolve => {
  87. resolve(
  88. LICENSE +
  89. contents.replace(
  90. 'parse: function parse',
  91. SQL_STATEMENTS_PARSER_JSDOC + 'parse: function parse'
  92. ) +
  93. 'export default sqlStatementsParser;\n'
  94. );
  95. })
  96. }
  97. };
  98. const mkdir = path =>
  99. new Promise((resolve, reject) => {
  100. if (fs.existsSync(path)) {
  101. resolve();
  102. } else {
  103. fs.mkdir(path, err => {
  104. if (err) {
  105. reject(err);
  106. }
  107. resolve();
  108. });
  109. }
  110. });
  111. const readFile = path =>
  112. new Promise((resolve, reject) => {
  113. fs.readFile(path, (err, buf) => {
  114. if (err) {
  115. reject(err);
  116. }
  117. resolve(buf ? buf.toString() : '');
  118. });
  119. });
  120. const writeFile = (path, contents) =>
  121. new Promise((resolve, reject) => {
  122. fs.writeFile(path, contents, err => {
  123. if (err) {
  124. reject();
  125. }
  126. resolve();
  127. });
  128. });
  129. const copyFile = (source, destination) =>
  130. new Promise((resolve, reject) => {
  131. readFile(source)
  132. .then(contents => {
  133. writeFile(destination, contents)
  134. .then(resolve)
  135. .catch(reject);
  136. })
  137. .catch(reject);
  138. });
  139. const deleteFile = path => {
  140. fs.unlinkSync(path);
  141. };
  142. const execCmd = cmd =>
  143. new Promise((resolve, reject) => {
  144. exec(cmd, (err, stdout, stderr) => {
  145. if (err) {
  146. reject('stderr:\n' + stderr + '\n\nstdout:\n' + stdout);
  147. }
  148. resolve(stdout);
  149. });
  150. });
  151. const generateParser = parserName =>
  152. new Promise((resolve, reject) => {
  153. const parserConfig = parserDefinitions[parserName];
  154. const concatPromise = new Promise((resolve, reject) => {
  155. if (parserConfig.sources.length > 1 && parserConfig.target) {
  156. console.log('Concatenating files...');
  157. const promises = parserConfig.sources.map(fileName => readFile(JISON_FOLDER + fileName));
  158. Promise.all(promises)
  159. .then(contents => {
  160. writeFile(JISON_FOLDER + parserConfig.target, contents.join('')).then(() => {
  161. resolve(JISON_FOLDER + parserConfig.target);
  162. });
  163. })
  164. .catch(reject);
  165. } else if (parserConfig.sources.length === 1) {
  166. resolve(JISON_FOLDER + parserConfig.sources[0]);
  167. } else {
  168. reject('No jison source specified');
  169. }
  170. });
  171. concatPromise
  172. .then(targetPath => {
  173. let jisonCommand = 'jison ' + targetPath;
  174. if (parserConfig.lexer) {
  175. jisonCommand += ' ' + JISON_FOLDER + parserConfig.lexer;
  176. }
  177. jisonCommand += ' -m js';
  178. console.log('Generating parser...');
  179. execCmd(jisonCommand)
  180. .then(stdout => {
  181. if (/\S/.test(stdout)) {
  182. console.log('got output for: ' + jisonCommand);
  183. console.log(stdout);
  184. }
  185. if (parserConfig.sources.length > 1) {
  186. deleteFile(targetPath); // Remove concatenated file
  187. }
  188. console.log('Adjusting JS...');
  189. const generatedJsFileName = parserConfig.target
  190. .replace('.jison', '.js')
  191. .replace(/^.*\/([^/]+)$/, '$1');
  192. readFile(generatedJsFileName)
  193. .then(contents => {
  194. parserConfig
  195. .afterParse(contents)
  196. .then(finalContents => {
  197. writeFile(parserConfig.outputFolder + generatedJsFileName, finalContents)
  198. .then(() => {
  199. deleteFile(generatedJsFileName);
  200. resolve();
  201. })
  202. .catch(reject);
  203. })
  204. .catch(reject);
  205. })
  206. .catch(reject);
  207. })
  208. .catch(reject);
  209. })
  210. .catch(reject);
  211. });
  212. let parsersToGenerate = [];
  213. const invalid = [];
  214. let all = false;
  215. const listDir = folder =>
  216. new Promise(resolve => {
  217. fs.readdir(folder, (err, files) => {
  218. resolve(files);
  219. });
  220. });
  221. const findParser = (fileIndex, folder, sharedFiles, autocomplete) => {
  222. const prefix = autocomplete ? 'autocomplete' : 'syntax';
  223. if (fileIndex[prefix + '_header.jison'] && fileIndex[prefix + '_footer.jison']) {
  224. const parserName = folder + (autocomplete ? 'AutocompleteParser' : 'SyntaxParser');
  225. const parserDefinition = {
  226. sources: ['sql/' + folder + '/' + prefix + '_header.jison'].concat(sharedFiles),
  227. lexer: 'sql/' + folder + '/sql.jisonlex',
  228. target: 'sql/' + folder + '/' + parserName + '.jison',
  229. sqlParser: autocomplete ? 'AUTOCOMPLETE' : 'SYNTAX',
  230. outputFolder: 'desktop/core/src/desktop/js/parse/sql/' + folder + '/',
  231. afterParse: contents =>
  232. new Promise(resolve => {
  233. resolve(
  234. LICENSE +
  235. contents
  236. .replace(
  237. 'var ' + parserName + ' = ',
  238. "import SqlParseSupport from 'parse/sql/" +
  239. folder +
  240. "/sqlParseSupport';\n\nvar " +
  241. parserName +
  242. ' = '
  243. )
  244. .replace(
  245. 'loc: yyloc,',
  246. "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),"
  247. ) +
  248. '\nexport default ' +
  249. parserName +
  250. ';\n'
  251. );
  252. })
  253. };
  254. parserDefinition.sources.push('sql/' + folder + '/' + prefix + '_footer.jison');
  255. parserDefinitions[parserName] = parserDefinition;
  256. } else {
  257. console.log(
  258. "Warn: Could not find '" +
  259. prefix +
  260. "_header.jison' or '" +
  261. prefix +
  262. "_footer.jison' in " +
  263. JISON_FOLDER +
  264. 'sql/' +
  265. folder +
  266. '/'
  267. );
  268. }
  269. };
  270. const identifySqlParsers = () =>
  271. new Promise(resolve => {
  272. listDir(JISON_FOLDER + 'sql').then(files => {
  273. const promises = [];
  274. files.forEach(folder => {
  275. promises.push(
  276. listDir(JISON_FOLDER + 'sql/' + folder).then(jisonFiles => {
  277. const fileIndex = {};
  278. jisonFiles.forEach(jisonFile => {
  279. fileIndex[jisonFile] = true;
  280. });
  281. const sharedFiles = jisonFiles
  282. .filter(jisonFile => jisonFile.indexOf('sql_') !== -1)
  283. .map(jisonFile => 'sql/' + folder + '/' + jisonFile);
  284. if (fileIndex['sql.jisonlex']) {
  285. findParser(fileIndex, folder, sharedFiles, true);
  286. findParser(
  287. fileIndex,
  288. folder,
  289. sharedFiles.filter(path => path.indexOf('_error.jison') === -1),
  290. false
  291. );
  292. } else {
  293. console.log(
  294. "Warn: Could not find 'sql.jisonlex' in " + JISON_FOLDER + 'sql/' + folder + '/'
  295. );
  296. }
  297. })
  298. );
  299. });
  300. Promise.all(promises).then(resolve);
  301. });
  302. });
  303. const prepareForNewParser = () =>
  304. new Promise((resolve, reject) => {
  305. if (process.argv.length === 3 && process.argv[0] === '-new') {
  306. process.argv.shift();
  307. const source = process.argv.shift();
  308. const target = process.argv.shift();
  309. console.log("Generating new parser '" + target + "' based on '" + source + "'...");
  310. process.argv.push(target);
  311. if (
  312. !Object.keys(parserDefinitions).some(key => {
  313. if (key.indexOf(source) === 0) {
  314. mkdir(JISON_FOLDER + 'sql/' + target)
  315. .then(() => {
  316. listDir(JISON_FOLDER + 'sql/' + source).then(files => {
  317. const copyPromises = [];
  318. files.forEach(file => {
  319. copyPromises.push(
  320. copyFile(
  321. JISON_FOLDER + 'sql/' + source + '/' + file,
  322. JISON_FOLDER + 'sql/' + target + '/' + file
  323. )
  324. );
  325. });
  326. Promise.all(copyPromises).then(() => {
  327. const autocompleteSources = ['sql/' + target + '/autocomplete_header.jison'];
  328. const syntaxSources = ['sql/' + target + '/syntax_header.jison'];
  329. const lexer = 'sql/' + target + '/sql.jisonlex';
  330. files.forEach(file => {
  331. if (file.indexOf('sql_') === 0) {
  332. autocompleteSources.push('sql/' + target + '/' + file);
  333. syntaxSources.push('sql/' + target + '/' + file);
  334. }
  335. });
  336. autocompleteSources.push('sql/' + target + '/autocomplete_footer.jison');
  337. syntaxSources.push('sql/' + target + '/syntax_footer.jison');
  338. mkdir('desktop/core/src/desktop/js/parse/sql/' + target).then(() => {
  339. readFile(
  340. 'desktop/core/src/desktop/js/parse/sql/' + source + '/sqlParseSupport.js'
  341. ).then(parseSupportContents => {
  342. parseSupportContents = parseSupportContents.replace(
  343. /parser\.yy\.activeDialect = '[^']+';'/g,
  344. "parser.yy.activeDialect = '" + target + "';"
  345. );
  346. writeFile(
  347. 'desktop/core/src/desktop/js/parse/sql/' + target + '/sqlParseSupport.js',
  348. parseSupportContents
  349. ).then(() => {
  350. parserDefinitions[target + 'AutocompleteParser'] = {
  351. sources: autocompleteSources,
  352. lexer: lexer,
  353. target: 'sql/' + target + '/' + target + 'AutocompleteParser.jison',
  354. sqlParser: 'AUTOCOMPLETE',
  355. outputFolder: 'desktop/core/src/desktop/js/parse/sql/' + target + '/',
  356. afterParse: contents =>
  357. new Promise(resolveAfterParse => {
  358. resolveAfterParse(
  359. LICENSE +
  360. contents
  361. .replace(
  362. 'var ' + target + 'AutocompleteParser = ',
  363. "import SqlParseSupport from 'parse/sql/" +
  364. target +
  365. "/sqlParseSupport';\n\nvar " +
  366. target +
  367. 'AutocompleteParser = '
  368. )
  369. .replace(
  370. 'loc: yyloc,',
  371. "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),"
  372. ) +
  373. '\nexport default ' +
  374. target +
  375. 'AutocompleteParser;\n'
  376. );
  377. })
  378. };
  379. parserDefinitions[target + 'SyntaxParser'] = {
  380. sources: syntaxSources,
  381. lexer: lexer,
  382. target: 'sql/' + target + '/' + target + 'SyntaxParser.jison',
  383. sqlParser: 'SYNTAX',
  384. outputFolder: 'desktop/core/src/desktop/js/parse/sql/' + target + '/',
  385. afterParse: contents =>
  386. new Promise(resolveAfterParse => {
  387. resolveAfterParse(
  388. LICENSE +
  389. contents
  390. .replace(
  391. 'var ' + target + 'SyntaxParser = ',
  392. "import SqlParseSupport from 'parse/sql/" +
  393. target +
  394. "/sqlParseSupport';\n\nvar " +
  395. target +
  396. 'SyntaxParser = '
  397. )
  398. .replace(
  399. 'loc: yyloc,',
  400. "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),"
  401. ) +
  402. '\nexport default ' +
  403. target +
  404. 'SyntaxParser;\n'
  405. );
  406. })
  407. };
  408. console.log(parserDefinitions);
  409. resolve();
  410. });
  411. });
  412. });
  413. });
  414. });
  415. })
  416. .catch(err => {
  417. console.log(err);
  418. });
  419. return true;
  420. }
  421. })
  422. ) {
  423. reject("No existing parser found for '" + source + "'");
  424. }
  425. } else {
  426. resolve();
  427. }
  428. });
  429. identifySqlParsers().then(() => {
  430. process.argv.shift();
  431. process.argv.shift();
  432. prepareForNewParser().then(() => {
  433. process.argv.forEach(arg => {
  434. if (arg === 'all') {
  435. all = true;
  436. } else if (parserDefinitions[arg]) {
  437. parsersToGenerate.push(arg);
  438. } else {
  439. let prefixFound = false;
  440. Object.keys(parserDefinitions).forEach(key => {
  441. if (key.indexOf(arg) === 0) {
  442. prefixFound = true;
  443. parsersToGenerate.push(key);
  444. }
  445. });
  446. if (!prefixFound) {
  447. invalid.push(arg);
  448. }
  449. }
  450. });
  451. if (all) {
  452. parsersToGenerate = Object.keys(parserDefinitions);
  453. }
  454. if (invalid.length) {
  455. console.log("No parser config found for: '" + invalid.join("', '") + "'");
  456. console.log(
  457. '\nPossible options are:\n ' +
  458. ['all'].concat(Object.keys(parserDefinitions)).join('\n ') +
  459. '\n'
  460. );
  461. return;
  462. }
  463. const parserCount = parsersToGenerate.length;
  464. let idx = 0;
  465. const generateRecursive = () => {
  466. idx++;
  467. if (parsersToGenerate.length) {
  468. const parserName = parsersToGenerate.pop();
  469. if (parserCount > 1) {
  470. console.log("Generating '" + parserName + "' (" + idx + '/' + parserCount + ')...');
  471. } else {
  472. console.log("Generating '" + parserName + "'...");
  473. }
  474. generateParser(parserName)
  475. .then(generateRecursive)
  476. .catch(error => {
  477. console.log(error);
  478. console.log('FAIL!');
  479. });
  480. } else {
  481. const autocompParsers = [];
  482. const syntaxParsers = [];
  483. console.log('Updating sqlParserRepository.js...');
  484. Object.keys(parserDefinitions).forEach(key => {
  485. if (parserDefinitions[key].sqlParser === 'AUTOCOMPLETE') {
  486. autocompParsers.push(
  487. AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE.replace(
  488. /KEY/g,
  489. key.replace('AutocompleteParser', '')
  490. )
  491. );
  492. } else if (parserDefinitions[key].sqlParser === 'SYNTAX') {
  493. syntaxParsers.push(
  494. SYNTAX_PARSER_IMPORT_TEMPLATE.replace(/KEY/g, key.replace('SyntaxParser', ''))
  495. );
  496. }
  497. });
  498. readFile(SQL_PARSER_REPOSITORY_PATH).then(contents => {
  499. contents = contents.replace(
  500. /const SYNTAX_MODULES = [^}]+}/,
  501. 'const SYNTAX_MODULES = {\n' + syntaxParsers.sort().join(',\n') + '\n}'
  502. );
  503. contents = contents.replace(
  504. /const AUTOCOMPLETE_MODULES = [^}]+}/,
  505. 'const AUTOCOMPLETE_MODULES = {\n' + autocompParsers.sort().join(',\n') + '\n}'
  506. );
  507. writeFile(SQL_PARSER_REPOSITORY_PATH, contents).then(() => {
  508. console.log('Done!\n');
  509. });
  510. });
  511. }
  512. };
  513. generateRecursive();
  514. });
  515. });