impalaExtractor.js 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. let fs = require('fs');
  17. let libxml = require('libxmljs');
  18. let keyDefs = {};
  19. let pathToXref = {};
  20. let knownTitles = {};
  21. let xrefs = {};
  22. let conRefs = {};
  23. let handleChildNodes = (x, path, body) => {
  24. x.childNodes().forEach(y => {
  25. handleElement(y, path, body);
  26. });
  27. };
  28. let addStartElement = (x, path, body, elemName, classes) => {
  29. let start = '<' + elemName;
  30. if (x.attr('id')) {
  31. let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value();
  32. start += ' id="' + id + '"';
  33. }
  34. if (classes) {
  35. start += ' class="' + classes + '"';
  36. }
  37. start += '>';
  38. body.push(start);
  39. };
  40. let wrapHtmlElement = (x, path, body, elemName, classes) => {
  41. addStartElement(x, path, body, elemName, classes);
  42. handleChildNodes(x, path, body);
  43. body.push('</' + elemName + '>');
  44. };
  45. let handleElement = (x, path, body) => {
  46. if (x.name() === 'text') {
  47. if (x.text().trim()) {
  48. body.push(x.text());
  49. }
  50. } else if (x.name() === 'xref') {
  51. body.push({
  52. xrefNode: x,
  53. path: path
  54. });
  55. } else {
  56. if (!x.childNodes().length) {
  57. if (x.name() === 'keyword' && x.attr('keyref')) {
  58. if (keyDefs[x.attr('keyref').value()]) {
  59. body.push(keyDefs[x.attr('keyref').value()]);
  60. } else {
  61. body.push(x.attr('keyref').value())
  62. }
  63. } else if (x.attr('conref') && x.attr('conref').value().indexOf('impala_common.xml') !== -1) {
  64. var id = x.attr('conref').value().replace(/^.*common\//, '');
  65. if (conRefs[id]) {
  66. handleElement(conRefs[id], path, body);
  67. } else {
  68. console.log('concept ref not found with id: ' + id);
  69. }
  70. }
  71. } else {
  72. switch (x.name()) {
  73. case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; }
  74. case 'p': {
  75. wrapHtmlElement(x, path, body, 'p');
  76. break;
  77. }
  78. case 'concept': {
  79. if (!x.attr('audience') || x.attr('audience').value() !== 'hidden') {
  80. wrapHtmlElement(x, path, body, 'div');
  81. if (x.attr('id') && x.get('title')) {
  82. var titleParts = [];
  83. handleElement(x.get('title'), path, titleParts);
  84. knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join('');
  85. }
  86. }
  87. break;
  88. }
  89. case 'conbody': {
  90. wrapHtmlElement(x, path, body, 'div');
  91. if (x.parent().get('title')) {
  92. var titleParts = [];
  93. handleElement(x.parent().get('title'), path, titleParts);
  94. knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join('');
  95. }
  96. break;
  97. }
  98. case 'codeph':
  99. case 'cmdname':
  100. case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; }
  101. case 'codeblock': {
  102. addStartElement(x, path, body, 'div', 'sql-docs-code-block');
  103. var preChildren = [];
  104. handleChildNodes(x, path, preChildren);
  105. preChildren.forEach(function (child) {
  106. if (typeof child === 'string') {
  107. body.push(child.replace(/\n/g, '<br/>'));
  108. } else if (child.xrefNode) {
  109. body.push(child);
  110. } else {
  111. console.log('Could not process codeblock child: ' + child.toString());
  112. }
  113. });
  114. body.push('</div>');
  115. break;
  116. }
  117. case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; }
  118. case 'varname':
  119. case 'filepath':
  120. case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; }
  121. case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; }
  122. case 'example': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-example'); break; }
  123. case 'b':
  124. case 'dl':
  125. case 'dlentry':
  126. case 'ol':
  127. case 'dd':
  128. case 'dt':
  129. case 'q':
  130. case 'i':
  131. case 'sup':
  132. case 'ul':
  133. case 'li': { wrapHtmlElement(x, path, body, x.name()); break; }
  134. case 'indexterm':
  135. case 'metadata':
  136. case 'fig':
  137. case 'prolog':
  138. case 'titlealts':
  139. case 'uicontrol':
  140. case 'table':
  141. case 'navtitle': break;
  142. default: console.log('Could not process element of type: ' + x.name() + ' in ' + path);
  143. }
  144. }
  145. }
  146. };
  147. let parseDml = (path) => {
  148. return new Promise((resolve) => {
  149. fs.readFile(path, 'utf8', (err, data) => {
  150. try {
  151. let xmlDoc = libxml.parseXmlString(data);
  152. let body = [];
  153. var titleParts = [];
  154. handleChildNodes(xmlDoc.get('//title'), path, titleParts);
  155. if (xmlDoc.root().attr('id')) {
  156. knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join('');
  157. }
  158. xmlDoc.get('//title').remove();
  159. xmlDoc.childNodes().forEach(x => {
  160. handleElement(x, path, body);
  161. });
  162. resolve({ title: titleParts.join(''), body: body});
  163. } catch (err) {
  164. console.log(path);
  165. console.log(err);
  166. }
  167. });
  168. });
  169. };
  170. let flattenBody = (body, prefix) => {
  171. let bodyString = '';
  172. body.forEach(function (bodyElement) {
  173. if (typeof bodyElement === 'string') {
  174. bodyString += bodyElement;
  175. } else if (bodyElement.xrefNode) {
  176. if (bodyElement.xrefNode.attr('href')) {
  177. if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') {
  178. bodyString += '<a target="_blank" href="' + bodyElement.xrefNode.attr('href').value() + '">' + bodyElement.xrefNode.text() + '</a>'
  179. } else {
  180. let href = bodyElement.xrefNode.attr('href').value();
  181. if (href.indexOf('#') === 0) {
  182. href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href;
  183. } else if (href.indexOf('topics/') !== -1) {
  184. href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc.
  185. } else {
  186. href = 'topics/' + href;
  187. }
  188. var split = href.split('#');
  189. var unknown = false;
  190. let title = href;
  191. if (knownTitles[href]) {
  192. title = bodyElement.xrefNode.text() || knownTitles[href];
  193. } else if (knownTitles[split[0]]) {
  194. title = bodyElement.xrefNode.text() || knownTitles[split[0]];
  195. } else if (bodyElement.xrefNode.text()) {
  196. unknown = true;
  197. title = bodyElement.xrefNode.text();
  198. } else if (split[1]) {
  199. unknown = true;
  200. title = split[1].replace(/_/g, ' ');
  201. } else {
  202. unknown = true;
  203. title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' ');
  204. }
  205. if (unknown) {
  206. bodyString += '<span>' + title + '</span>'; // Unknown = not parsed reference as some docs are excluded
  207. } else {
  208. bodyString += '<a href="javascript: void(0);" class="lang-ref-link" data-target="' + href + '">' + title + '</a>';
  209. }
  210. }
  211. }
  212. }
  213. });
  214. return bodyString;
  215. };
  216. let stringifyTopic = (topic, prefix) => {
  217. let result = prefix + '{\n' + prefix + ' id: \'' + topic.ref + '\',\n' + prefix + ' title: \'' + topic.title + '\',\n' + prefix + ' weight: 1,\n' + prefix + ' bodyMatch: ko.observable(),\n' + prefix + ' open: ko.observable(false),\n' + prefix +' titleMatch: ko.observable()';
  218. if (topic.body.length) {
  219. result += ',\n' + prefix + ' body: \'' + flattenBody(topic.body, prefix) + '\''
  220. }
  221. if (topic.children.length) {
  222. result += ',\n' + prefix + ' children: [\n';
  223. let stringifiedChildren = [];
  224. topic.children.forEach(child => {
  225. stringifiedChildren.push(stringifyTopic(child, prefix + ' '))
  226. });
  227. result += stringifiedChildren.join(',\n');
  228. result += prefix + ']';
  229. } else {
  230. result += ',\n' + prefix + ' children: []\n';
  231. }
  232. result += prefix + '}';
  233. return result;
  234. };
  235. class Topic {
  236. constructor (ref, node, promises) {
  237. this.ref = ref;
  238. this.path = '../Impala/docs/' + ref;
  239. this.children = [];
  240. this.title = '';
  241. this.body = [];
  242. if (pathToXref[this.ref]) {
  243. pathToXref[this.ref].parsed = true;
  244. }
  245. promises.push(parseDml(this.path).then(parseResult => {
  246. this.title = parseResult.title;
  247. this.body = parseResult.body;
  248. }));
  249. if (node.childNodes().length) {
  250. node.childNodes().forEach(x => {
  251. if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
  252. this.children.push(new Topic(x.attr('href').value(), x, promises));
  253. }
  254. });
  255. }
  256. }
  257. toJson() {
  258. return JSON.stringify({
  259. body: flattenBody(this.body, ''),
  260. title: this.title
  261. })
  262. }
  263. }
  264. fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => {
  265. if (err) {
  266. console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)');
  267. console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder');
  268. return;
  269. }
  270. libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => {
  271. if (x.name() === 'keydef' && x.attr('keys')) {
  272. let valNode = x.get('topicmeta/keywords/keyword');
  273. if (valNode) {
  274. keyDefs[x.attr('keys').value()] = valNode.text();
  275. } else if (x.attr('href')) {
  276. xrefs[x.attr('keys')] = {
  277. ref: x.attr('href').value(),
  278. parsed: false,
  279. external: x.attr('scope') && x.attr('scope').value() === 'external'
  280. };
  281. pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')];
  282. }
  283. }
  284. });
  285. fs.readFile('../Impala/docs/shared/impala_common.xml', 'utf-8', (err, commonRaw) => {
  286. let handleCommonChildren = children => {
  287. children.forEach(child => {
  288. if (child.attr('id')) {
  289. conRefs[child.attr('id').value()] = child;
  290. }
  291. if (child.childNodes().length) {
  292. handleCommonChildren(child.childNodes());
  293. }
  294. })
  295. };
  296. handleCommonChildren(libxml.parseXmlString(commonRaw).get('//conbody').childNodes());
  297. fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => {
  298. let topics = [];
  299. let promises = [];
  300. libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => {
  301. if (x.name() === 'topicref') {
  302. topics.push(new Topic(x.attr('href').value(), x, promises));
  303. }
  304. });
  305. let index = {};
  306. let topLevel = [];
  307. Promise.all(promises).then(() => {
  308. let saveTopics = (topics, parent) => {
  309. topics.forEach(topic => {
  310. let entry = {
  311. title: topic.title,
  312. ref: topic.ref,
  313. children: []
  314. };
  315. if (!parent) {
  316. topLevel.push(entry)
  317. } else {
  318. parent.children.push(entry);
  319. }
  320. let fileName = topic.ref.replace('topics/', '').replace('.xml', '.json');
  321. index[topic.ref] = fileName;
  322. fs.writeFile('desktop/core/src/desktop/static/desktop/docs/impala/' + fileName, topic.toJson(), () => { });
  323. saveTopics(topic.children, entry);
  324. })
  325. };
  326. saveTopics(topics);
  327. fs.readFile('tools/sql-docs/impala_doc_index.mako.template', 'utf-8', (err, contents) => {
  328. let indexStrings = [];
  329. Object.keys(index).forEach(key => {
  330. indexStrings.push('\'' + key + '\':\'${ static(\'desktop/docs/impala/' + index[key] + '\') }\'')
  331. });
  332. contents = contents.replace('/* docIndex */', indexStrings.join(','));
  333. let createTopicJs = (entry) => {
  334. return '{title:\'' + entry.title +'\',ref:\'' + entry.ref + '\',children:[' + entry.children.map(createTopicJs).join(',') + ']}';
  335. };
  336. contents = contents.replace('/* topLevel */', topLevel.map(createTopicJs).join(','));
  337. fs.writeFile('desktop/core/src/desktop/templates/impala_doc_index.mako', contents, () => {
  338. console.log('desktop/core/src/desktop/templates/impala_doc_index.mako written.');
  339. })
  340. });
  341. });
  342. });
  343. })
  344. });