hiveExtractor.js 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. /* eslint-disable no-restricted-syntax */
  17. const program = require('commander');
  18. const EPub = require('epub');
  19. const Topic = require('./Topic');
  20. const libxml = require('libxmljs');
  21. const extractorUtils = require('./extractorUtils');
  22. const outputPath = './desktop/core/src/desktop/static/desktop/docs/hive/';
  23. const mako = './desktop/core/src/desktop/templates/sql_doc_index.mako';
  24. const jsonHandler = require('./jsonHandler');
  25. // Handle command line arguments
  26. program
  27. .version('1.0')
  28. .option('-e, --epub [path]', 'the path to the Hive epub file')
  29. .parse(process.argv);
  30. const epub = new EPub(program.epub);
  31. const convertToPre = (element, fragments) => {
  32. switch (element.name()) {
  33. case 'div':
  34. element.childNodes().forEach(node => {
  35. convertToPre(node, fragments);
  36. });
  37. break;
  38. case 'text':
  39. if (fragments.length === 1) {
  40. fragments.push(element.text().replace(/^\n/, ''));
  41. } else {
  42. fragments.push(element.text());
  43. }
  44. break;
  45. case 'code':
  46. if (
  47. element.attr('class') &&
  48. element
  49. .attr('class')
  50. .value()
  51. .indexOf('value') !== -1
  52. ) {
  53. fragments.push('<span class="hue-doc-varname">');
  54. element.childNodes().forEach(node => {
  55. convertToPre(node, fragments);
  56. });
  57. fragments.push('</span>');
  58. break;
  59. }
  60. default:
  61. element.childNodes().forEach(node => {
  62. convertToPre(node, fragments);
  63. });
  64. }
  65. };
  66. const adaptElement = element => {
  67. if (
  68. element.attr('class') &&
  69. element
  70. .attr('class')
  71. .value()
  72. .indexOf('syntaxhighlighter') !== -1
  73. ) {
  74. const fragments = ['<div class="hue-doc-codeblock">'];
  75. element.childNodes().forEach(childNode => {
  76. convertToPre(childNode, fragments);
  77. });
  78. fragments.push('</div>');
  79. const replacement = fragments.join('');
  80. element.replace(libxml.parseHtmlFragment(replacement).root());
  81. } else if (element.attr('class')) {
  82. if (
  83. element
  84. .attr('class')
  85. .value()
  86. .indexOf('admonition') !== -1
  87. ) {
  88. element.attr({ class: 'hue-doc-note-hive' });
  89. } else {
  90. element.attr('class').remove();
  91. }
  92. }
  93. switch (element.name()) {
  94. case 'a':
  95. if (
  96. element.attr('href') &&
  97. element
  98. .attr('href')
  99. .value()
  100. .indexOf('/links') == 0
  101. ) {
  102. // Internal link
  103. let ref = element
  104. .attr('href')
  105. .value()
  106. .replace(/\/links\/_[0-9]+\//, '');
  107. extractorUtils.removeAllAttributes(element);
  108. const anchorMatch = ref.match(/.html#(.+)$/);
  109. if (anchorMatch) {
  110. element.attr('data-doc-anchor-id', anchorMatch[1]);
  111. ref = ref.replace('#' + anchorMatch[1], '');
  112. }
  113. ref = '_' + ref.replace('.html', '');
  114. element.attr({
  115. class: 'hue-doc-internal-link',
  116. href: 'javascript:void(0);',
  117. 'data-doc-ref': ref
  118. });
  119. } else if (element.attr('href')) {
  120. // External link
  121. const href = element.attr('href').value();
  122. extractorUtils.removeAllAttributes(element);
  123. element.attr({ class: 'hue-doc-external-link', href: href, target: '_blank' });
  124. }
  125. break;
  126. case 'h1':
  127. element.text(element.text().replace(/LanguageManual\s(.+)/, '$1'));
  128. element.attr({ class: 'hue-doc-title-hive' });
  129. break;
  130. case 'table':
  131. case 'td':
  132. case 'th':
  133. case 'tr':
  134. element.attr({ class: 'hue-doc-' + element.name() + '-hive' });
  135. default:
  136. }
  137. element.childNodes().forEach(adaptElement);
  138. };
  139. epub.on('end', () => {
  140. const rootTopics = [];
  141. const lastTopicPerLevel = {};
  142. const promises = [];
  143. epub.flow.forEach(chapter => {
  144. promises.push(
  145. new Promise((resolve, reject) => {
  146. const topic = new Topic('/', chapter.id);
  147. topic.fragment = {
  148. title: {
  149. text: () => chapter.title.replace(/LanguageManual\s(.+)/, '$1')
  150. }
  151. };
  152. epub.getChapter(chapter.id, (error, text) => {
  153. try {
  154. const contents = libxml.parseHtmlFragment('<div>' + text + '</div>');
  155. topic.domXml = contents.root();
  156. adaptElement(topic.domXml);
  157. resolve();
  158. } catch (error) {
  159. reject();
  160. }
  161. });
  162. if (lastTopicPerLevel[chapter.level - 1]) {
  163. lastTopicPerLevel[chapter.level - 1].children.push(topic);
  164. }
  165. if (chapter.level === 0) {
  166. rootTopics.push(topic);
  167. }
  168. lastTopicPerLevel[chapter.level] = topic;
  169. })
  170. );
  171. });
  172. Promise.all(promises).then(() => {
  173. const rootTopic = rootTopics[0];
  174. rootTopic.children.forEach(childTopic => {
  175. rootTopics.push(childTopic);
  176. });
  177. rootTopic.children = [];
  178. jsonHandler
  179. .saveTopics(rootTopics, outputPath, mako, false)
  180. .then(() => {
  181. console.log('Done.');
  182. })
  183. .catch(() => {
  184. console.log('Fail.');
  185. });
  186. });
  187. });
  188. epub.parse();
  189. /* eslint-enable no-restricted-syntax */