impalaExtractor.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. let fs = require('fs');
  17. let libxml = require('libxmljs');
  18. let keyDefs = {};
  19. let pathToXref = {};
  20. let knownTitles = {};
  21. let xrefs = {};
  22. let conRefs = {};
  23. let handleChildNodes = (x, path, body) => {
  24. x.childNodes().forEach(y => {
  25. handleElement(y, path, body);
  26. });
  27. };
  28. let addStartElement = (x, path, body, elemName, classes) => {
  29. let start = '<' + elemName;
  30. if (x.attr('id')) {
  31. let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value();
  32. start += ' id="' + id + '"';
  33. }
  34. if (classes) {
  35. start += ' class="' + classes + '"';
  36. }
  37. start += '>';
  38. body.push(start);
  39. };
  40. let wrapHtmlElement = (x, path, body, elemName, classes) => {
  41. addStartElement(x, path, body, elemName, classes);
  42. handleChildNodes(x, path, body);
  43. body.push('</' + elemName + '> ');
  44. };
  45. let handleElement = (x, path, body) => {
  46. if (x.name() === 'text') {
  47. if (x.text().trim()) {
  48. body.push(x.text());
  49. }
  50. } else if (x.name() === 'xref') {
  51. body.push({
  52. xrefNode: x,
  53. path: path
  54. });
  55. } else {
  56. if (!x.childNodes().length) {
  57. if (x.name() === 'keyword' && x.attr('keyref')) {
  58. if (keyDefs[x.attr('keyref').value()]) {
  59. body.push(keyDefs[x.attr('keyref').value()]);
  60. } else {
  61. body.push(x.attr('keyref').value())
  62. }
  63. } else if (x.attr('conref') && x.attr('conref').value().indexOf('impala_common.xml') !== -1) {
  64. var id = x.attr('conref').value().replace(/^.*common\//, '');
  65. if (conRefs[id]) {
  66. handleElement(conRefs[id], path, body);
  67. } else {
  68. console.log('concept ref not found with id: ' + id);
  69. }
  70. }
  71. } else {
  72. switch (x.name()) {
  73. case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; }
  74. case 'p': {
  75. wrapHtmlElement(x, path, body, 'p');
  76. break;
  77. }
  78. case 'concept': {
  79. if (!x.attr('audience') || x.attr('audience').value() !== 'hidden') {
  80. wrapHtmlElement(x, path, body, 'div');
  81. if (x.attr('id') && x.get('title')) {
  82. var titleParts = [];
  83. handleElement(x.get('title'), path, titleParts);
  84. knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join('');
  85. }
  86. }
  87. break;
  88. }
  89. case 'conbody': {
  90. wrapHtmlElement(x, path, body, 'div');
  91. if (x.parent().get('title')) {
  92. var titleParts = [];
  93. handleElement(x.parent().get('title'), path, titleParts);
  94. knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join('');
  95. }
  96. break;
  97. }
  98. case 'codeph':
  99. case 'cmdname':
  100. case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; }
  101. case 'codeblock': {
  102. addStartElement(x, path, body, 'div', 'sql-docs-code-block');
  103. var preChildren = [];
  104. handleChildNodes(x, path, preChildren);
  105. preChildren.forEach(function (child) {
  106. body.push(child.replace(/^\s*/g, '').replace(/\n/g, '<br/>'));
  107. });
  108. body.push('</div>');
  109. break;
  110. }
  111. case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; }
  112. case 'varname':
  113. case 'filepath':
  114. case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; }
  115. case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; }
  116. case 'b': { wrapHtmlElement(x, path, body, 'b'); break; }
  117. case 'q': { wrapHtmlElement(x, path, body, 'q'); break; }
  118. case 'i': { wrapHtmlElement(x, path, body, 'i'); break; }
  119. case 'sup': { wrapHtmlElement(x, path, body, 'sup'); break; }
  120. case 'ul': { wrapHtmlElement(x, path, body, 'ul'); break; }
  121. case 'li': { wrapHtmlElement(x, path, body, 'li'); break; }
  122. case 'indexterm':
  123. case 'metadata':
  124. case 'fig':
  125. case 'prolog':
  126. case 'titlealts':
  127. case 'uicontrol':
  128. case 'table':
  129. case 'navtitle': break;
  130. default: console.log('Could not process element of type: ' + x.name() + ' in ' + path);
  131. }
  132. }
  133. }
  134. };
  135. let parseDml = (path) => {
  136. return new Promise((resolve) => {
  137. fs.readFile(path, 'utf8', (err, data) => {
  138. try {
  139. let xmlDoc = libxml.parseXmlString(data);
  140. let body = [];
  141. var titleParts = [];
  142. handleChildNodes(xmlDoc.get('//title'), path, titleParts);
  143. if (xmlDoc.root().attr('id')) {
  144. knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join('');
  145. }
  146. xmlDoc.get('//title').remove();
  147. xmlDoc.childNodes().forEach(x => {
  148. handleElement(x, path, body);
  149. });
  150. resolve({ title: titleParts.join(''), body: body});
  151. } catch (err) {
  152. console.log(path);
  153. console.log(err);
  154. }
  155. });
  156. });
  157. };
  158. class Topic {
  159. constructor (ref, node, promises) {
  160. this.ref = ref;
  161. this.path = '../Impala/docs/' + ref;
  162. this.children = [];
  163. this.title = '';
  164. this.body = [];
  165. if (pathToXref[this.ref]) {
  166. pathToXref[this.ref].parsed = true;
  167. }
  168. promises.push(parseDml(this.path).then(parseResult => {
  169. this.title = parseResult.title;
  170. this.body = parseResult.body;
  171. }));
  172. if (node.childNodes().length) {
  173. node.childNodes().forEach(x => {
  174. if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
  175. this.children.push(new Topic(x.attr('href').value(), x, promises));
  176. }
  177. });
  178. }
  179. }
  180. }
  181. fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => {
  182. if (err) {
  183. console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)');
  184. console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder');
  185. return;
  186. }
  187. libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => {
  188. if (x.name() === 'keydef' && x.attr('keys')) {
  189. let valNode = x.get('topicmeta/keywords/keyword');
  190. if (valNode) {
  191. keyDefs[x.attr('keys').value()] = valNode.text();
  192. } else if (x.attr('href')) {
  193. xrefs[x.attr('keys')] = {
  194. ref: x.attr('href').value(),
  195. parsed: false,
  196. external: x.attr('scope') && x.attr('scope').value() === 'external'
  197. };
  198. pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')];
  199. }
  200. }
  201. });
  202. let stringifyTopic = (topic, prefix) => {
  203. let result = prefix + '{\n' + prefix + ' id: \'' + topic.ref + '\',\n' + prefix + ' title: \'' + topic.title + '\',\n' + prefix + ' weight: 1,\n' + prefix + ' bodyMatch: ko.observable(),\n' + prefix + ' open: ko.observable(false),\n' + prefix +' titleMatch: ko.observable()';
  204. if (topic.body.length) {
  205. var bodyString = '';
  206. topic.body.forEach(function (bodyElement) {
  207. if (typeof bodyElement === 'string') {
  208. bodyString += bodyElement;
  209. } else if (bodyElement.xrefNode) {
  210. if (bodyElement.xrefNode.attr('href')) {
  211. if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') {
  212. bodyString += '<a target="_blank" href="' + bodyElement.xrefNode.attr('href').value() + '">' + bodyElement.xrefNode.text() + '</a>'
  213. } else {
  214. let href = bodyElement.xrefNode.attr('href').value();
  215. if (href.indexOf('#') === 0) {
  216. href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href;
  217. } else if (href.indexOf('topics/') !== -1) {
  218. href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc.
  219. } else {
  220. href = 'topics/' + href;
  221. }
  222. var split = href.split('#');
  223. var unknown = false;
  224. let title = href;
  225. if (knownTitles[href]) {
  226. title = bodyElement.xrefNode.text() || knownTitles[href];
  227. } else if (knownTitles[split[0]]) {
  228. title = bodyElement.xrefNode.text() || knownTitles[split[0]];
  229. } else if (bodyElement.xrefNode.text()) {
  230. unknown = true;
  231. title = bodyElement.xrefNode.text();
  232. } else if (split[1]) {
  233. unknown = true;
  234. title = split[1].replace(/_/g, ' ');
  235. } else {
  236. unknown = true;
  237. title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' ');
  238. }
  239. if (unknown) {
  240. bodyString += '<span>' + title + '</span>'; // Unknown = not parsed reference as some docs are excluded
  241. } else {
  242. bodyString += '<a href="javascript: void(0);" class="lang-ref-link" data-target="' + href + '">' + title + '</a>';
  243. }
  244. }
  245. }
  246. }
  247. });
  248. result += ',\n' + prefix + ' body: \'' + bodyString.replace(/([^\\])\\([^\\])/g, '$1\\\\$2').replace(/'/g, '\\\'').replace(/\n/g, '\' + \n' + prefix + ' \'') + '\''
  249. }
  250. if (topic.children.length) {
  251. result += ',\n' + prefix + ' children: [\n';
  252. let stringifiedChildren = [];
  253. topic.children.forEach(child => {
  254. stringifiedChildren.push(stringifyTopic(child, prefix + ' '))
  255. });
  256. result += stringifiedChildren.join(',\n');
  257. result += prefix + ']';
  258. } else {
  259. result += ',\n' + prefix + ' children: []\n';
  260. }
  261. result += prefix + '}';
  262. return result;
  263. };
  264. fs.readFile('../Impala/docs/shared/impala_common.xml', 'utf-8', (err, commonRaw) => {
  265. let handleCommonChildren = children => {
  266. children.forEach(child => {
  267. if (child.attr('id')) {
  268. conRefs[child.attr('id').value()] = child;
  269. }
  270. if (child.childNodes().length) {
  271. handleCommonChildren(child.childNodes());
  272. }
  273. })
  274. };
  275. handleCommonChildren(libxml.parseXmlString(commonRaw).get('//conbody').childNodes());
  276. fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => {
  277. let topics = [];
  278. let promises = [];
  279. libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => {
  280. if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
  281. topics.push(new Topic(x.attr('href').value(), x, promises));
  282. }
  283. });
  284. Promise.all(promises).then(() => {
  285. fs.readFile('tools/jison/license.txt', 'utf-8', (err, licenseHeader) => {
  286. let result = licenseHeader + '\n\n\n// NOTE: This is a generated file!\n// Run \'node tools/sql-docs/impalaExtractor.js\' to generate.\n\n\nvar impalaLangRefTopics = [\n';
  287. let stringifiedTopics = [];
  288. topics.forEach(topic => {
  289. stringifiedTopics.push(stringifyTopic(topic, ''));
  290. });
  291. result += stringifiedTopics.join(',\n');
  292. result += '\n];';
  293. fs.writeFile('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js', result, () => {
  294. console.log('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js updated!');
  295. })
  296. })
  297. });
  298. });
  299. })
  300. });