impalaExtractor.js 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. let fs = require('fs');
  17. let libxml = require('libxmljs');
  18. let keyDefs = {};
  19. let pathToXref = {};
  20. let knownTitles = {};
  21. let xrefs = {};
  22. let handleChildNodes = (x, path, body) => {
  23. x.childNodes().forEach(y => {
  24. handleElement(y, path, body);
  25. });
  26. };
  27. let addStartElement = (x, path, body, elemName, classes) => {
  28. let start = '<' + elemName;
  29. if (x.attr('id')) {
  30. let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value();
  31. start += ' id="' + id + '"';
  32. }
  33. if (classes) {
  34. start += ' class="' + classes + '"';
  35. }
  36. start += '>';
  37. body.push(start);
  38. };
  39. let wrapHtmlElement = (x, path, body, elemName, classes) => {
  40. addStartElement(x, path, body, elemName, classes);
  41. handleChildNodes(x, path, body);
  42. body.push('</' + elemName + '> ');
  43. };
  44. let handleElement = (x, path, body) => {
  45. if (x.name() === 'text') {
  46. if (x.text().trim()) {
  47. body.push(x.text());
  48. }
  49. } else if (x.name() === 'xref') {
  50. body.push({
  51. xrefNode: x,
  52. path: path
  53. });
  54. } else {
  55. if (!x.childNodes().length) {
  56. if (x.name() === 'keyword' && x.attr('keyref')) {
  57. if (keyDefs[x.attr('keyref').value()]) {
  58. body.push(keyDefs[x.attr('keyref').value()]);
  59. } else {
  60. body.push(x.attr('keyref').value())
  61. }
  62. }
  63. } else {
  64. switch (x.name()) {
  65. case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; }
  66. case 'p': { wrapHtmlElement(x, path, body, 'p'); break; }
  67. case 'concept': {
  68. wrapHtmlElement(x, path, body, 'div');
  69. if (x.attr('id') && x.get('title')) {
  70. var titleParts = [];
  71. handleElement(x.get('title'), path, titleParts);
  72. knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join('');
  73. }
  74. break;
  75. }
  76. case 'conbody': {
  77. wrapHtmlElement(x, path, body, 'div');
  78. if (x.parent().get('title')) {
  79. var titleParts = [];
  80. handleElement(x.parent().get('title'), path, titleParts);
  81. knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join('');
  82. }
  83. break;
  84. }
  85. case 'codeph':
  86. case 'cmdname':
  87. case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; }
  88. case 'codeblock': {
  89. addStartElement(x, path, body, 'div', 'sql-docs-code-block');
  90. var preChildren = [];
  91. handleChildNodes(x, path, preChildren);
  92. preChildren.forEach(function (child) {
  93. body.push(child.replace(/^\s*/g, '').replace(/\n/g, '<br/>'));
  94. });
  95. body.push('</div>');
  96. break;
  97. }
  98. case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; }
  99. case 'varname':
  100. case 'filepath':
  101. case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; }
  102. case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; }
  103. case 'b': { wrapHtmlElement(x, path, body, 'b', 'sql-docs-variable'); break; }
  104. case 'q': { wrapHtmlElement(x, path, body, 'q', 'sql-docs-variable'); break; }
  105. case 'i': { wrapHtmlElement(x, path, body, 'i', 'sql-docs-variable'); break; }
  106. case 'sup': { wrapHtmlElement(x, path, body, 'sup', 'sql-docs-variable'); break; }
  107. case 'ul': { wrapHtmlElement(x, path, body, 'ul', 'sql-docs-variable'); break; }
  108. case 'li': { wrapHtmlElement(x, path, body, 'li', 'sql-docs-variable'); break; }
  109. case 'indexterm':
  110. case 'metadata':
  111. case 'fig':
  112. case 'prolog':
  113. case 'navtitle': break;
  114. default: console.log('Could not process element of type: ' + x.name() + ' in ' + path);
  115. }
  116. }
  117. }
  118. };
  119. let parseDml = (path) => {
  120. return new Promise((resolve) => {
  121. fs.readFile(path, 'utf8', (err, data) => {
  122. try {
  123. let xmlDoc = libxml.parseXmlString(data);
  124. let body = [];
  125. var titleParts = [];
  126. handleChildNodes(xmlDoc.get('//title'), path, titleParts);
  127. if (xmlDoc.root().attr('id')) {
  128. knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join('');
  129. }
  130. xmlDoc.get('//title').remove();
  131. xmlDoc.childNodes().forEach(x => {
  132. handleChildNodes(x, path, body);
  133. });
  134. resolve({ title: titleParts.join(''), body: body});
  135. } catch (err) {
  136. console.log(path);
  137. console.log(err);
  138. }
  139. });
  140. });
  141. };
  142. class Topic {
  143. constructor (ref, node, promises) {
  144. this.ref = ref;
  145. this.path = '../Impala/docs/' + ref;
  146. this.children = [];
  147. this.title = '';
  148. this.body = [];
  149. if (pathToXref[this.ref]) {
  150. pathToXref[this.ref].parsed = true;
  151. }
  152. promises.push(parseDml(this.path).then(parseResult => {
  153. this.title = parseResult.title;
  154. this.body = parseResult.body;
  155. }));
  156. if (node.childNodes().length) {
  157. node.childNodes().forEach(x => {
  158. if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
  159. this.children.push(new Topic(x.attr('href').value(), x, promises));
  160. }
  161. });
  162. }
  163. }
  164. }
  165. fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => {
  166. if (err) {
  167. console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)');
  168. console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder');
  169. return;
  170. }
  171. libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => {
  172. if (x.name() === 'keydef' && x.attr('keys')) {
  173. let valNode = x.get('topicmeta/keywords/keyword');
  174. if (valNode) {
  175. keyDefs[x.attr('keys').value()] = valNode.text();
  176. } else if (x.attr('href')) {
  177. xrefs[x.attr('keys')] = {
  178. ref: x.attr('href').value(),
  179. parsed: false,
  180. external: x.attr('scope') && x.attr('scope').value() === 'external'
  181. };
  182. pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')];
  183. }
  184. }
  185. });
  186. let stringifyTopic = (topic, prefix) => {
  187. let result = prefix + '{\n' + prefix + ' id: \'' + topic.ref + '\',\n' + prefix + ' title: \'' + topic.title + '\',\n' + prefix + ' weight: 1,\n' + prefix + ' bodyMatch: ko.observable(),\n' + prefix + ' open: ko.observable(false),\n' + prefix +' titleMatch: ko.observable()';
  188. if (topic.body.length) {
  189. var bodyString = '';
  190. topic.body.forEach(function (bodyElement) {
  191. if (typeof bodyElement === 'string') {
  192. bodyString += bodyElement;
  193. } else if (bodyElement.xrefNode) {
  194. if (bodyElement.xrefNode.attr('href')) {
  195. if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') {
  196. bodyString += '<a target="_blank" href="' + bodyElement.xrefNode.attr('href').value() + '">' + bodyElement.xrefNode.text() + '</a>'
  197. } else {
  198. let href = bodyElement.xrefNode.attr('href').value();
  199. if (href.indexOf('#') === 0) {
  200. href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href;
  201. } else if (href.indexOf('topics/') !== -1) {
  202. href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc.
  203. } else {
  204. href = 'topics/' + href;
  205. }
  206. var split = href.split('#');
  207. var unknown = false;
  208. let title = href;
  209. if (knownTitles[href]) {
  210. title = bodyElement.xrefNode.text() || knownTitles[href];
  211. } else if (knownTitles[split[0]]) {
  212. title = bodyElement.xrefNode.text() || knownTitles[split[0]];
  213. } else if (bodyElement.xrefNode.text()) {
  214. unknown = true;
  215. title = bodyElement.xrefNode.text();
  216. } else if (split[1]) {
  217. unknown = true;
  218. title = split[1].replace(/_/g, ' ');
  219. } else {
  220. unknown = true;
  221. title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' ');
  222. }
  223. if (unknown) {
  224. bodyString += '<span>"' + title + '"</span>'; // Unknown = not parsed reference as some docs are excluded
  225. } else {
  226. bodyString += '<a href="javascript: void(0);" class="lang-ref-link" data-target="' + href + '">' + title + '</a>';
  227. }
  228. }
  229. }
  230. }
  231. });
  232. result += ',\n' + prefix + ' body: \'' + bodyString.replace(/([^\\])\\([^\\])/g, '$1\\\\$2').replace(/'/g, '\\\'').replace(/\n/g, '\' + \n' + prefix + ' \'') + '\''
  233. }
  234. if (topic.children.length) {
  235. result += ',\n' + prefix + ' children: [\n';
  236. let stringifiedChildren = [];
  237. topic.children.forEach(child => {
  238. stringifiedChildren.push(stringifyTopic(child, prefix + ' '))
  239. });
  240. result += stringifiedChildren.join(',\n');
  241. result += prefix + ']';
  242. } else {
  243. result += ',\n' + prefix + ' children: []\n';
  244. }
  245. result += prefix + '}';
  246. return result;
  247. };
  248. fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => {
  249. let topics = [];
  250. let promises = [];
  251. libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => {
  252. if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
  253. topics.push(new Topic(x.attr('href').value(), x, promises));
  254. }
  255. });
  256. Promise.all(promises).then(() => {
  257. fs.readFile('tools/jison/license.txt', 'utf-8', (err, licenseHeader) => {
  258. let result = licenseHeader + '\n\n\n// NOTE: This is a generated file!\n// Run \'node tools/sql-docs/impalaExtractor.js\' to generate.\n\n\nvar impalaLangRefTopics = [\n';
  259. let stringifiedTopics = [];
  260. topics.forEach(topic => {
  261. stringifiedTopics.push(stringifyTopic(topic, ''));
  262. });
  263. result += stringifiedTopics.join(',\n');
  264. result += '\n];';
  265. fs.writeFile('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js', result, () => {
  266. console.log('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js updated!');
  267. })
  268. })
  269. });
  270. });
  271. });