hiveExtractor.js 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. const program = require('commander');
  17. const EPub = require('epub');
  18. const Topic = require('./Topic');
  19. const libxml = require('libxmljs');
  20. const extractorUtils = require('./extractorUtils');
  21. const outputPath = './desktop/core/src/desktop/static/desktop/docs/hive/';
  22. const mako = './desktop/core/src/desktop/templates/sql_doc_index.mako';
  23. const jsonHandler = require('./jsonHandler');
  24. // Handle command line arguments
  25. program
  26. .version('1.0')
  27. .option('-e, --epub [path]', 'the path to the Hive epub file')
  28. .parse(process.argv);
  29. const epub = new EPub(program.epub);
  30. const convertToPre = (element, fragments) => {
  31. switch (element.name()) {
  32. case 'div':
  33. element.childNodes().forEach(node => {
  34. convertToPre(node, fragments);
  35. });
  36. break;
  37. case 'text':
  38. if (fragments.length === 1) {
  39. fragments.push(element.text().replace(/^\n/, ''));
  40. } else {
  41. fragments.push(element.text());
  42. }
  43. break;
  44. case 'code':
  45. if (
  46. element.attr('class') &&
  47. element
  48. .attr('class')
  49. .value()
  50. .indexOf('value') !== -1
  51. ) {
  52. fragments.push('<span class="hue-doc-varname">');
  53. element.childNodes().forEach(node => {
  54. convertToPre(node, fragments);
  55. });
  56. fragments.push('</span>');
  57. break;
  58. }
  59. default:
  60. element.childNodes().forEach(node => {
  61. convertToPre(node, fragments);
  62. });
  63. }
  64. };
  65. const adaptElement = element => {
  66. if (
  67. element.attr('class') &&
  68. element
  69. .attr('class')
  70. .value()
  71. .indexOf('syntaxhighlighter') !== -1
  72. ) {
  73. const fragments = ['<div class="hue-doc-codeblock">'];
  74. element.childNodes().forEach(childNode => {
  75. convertToPre(childNode, fragments);
  76. });
  77. fragments.push('</div>');
  78. const replacement = fragments.join('');
  79. element.replace(libxml.parseHtmlFragment(replacement).root());
  80. } else if (element.attr('class')) {
  81. if (
  82. element
  83. .attr('class')
  84. .value()
  85. .indexOf('admonition') !== -1
  86. ) {
  87. element.attr({ class: 'hue-doc-note-hive' });
  88. } else {
  89. element.attr('class').remove();
  90. }
  91. }
  92. switch (element.name()) {
  93. case 'a':
  94. if (
  95. element.attr('href') &&
  96. element
  97. .attr('href')
  98. .value()
  99. .indexOf('/links') == 0
  100. ) {
  101. // Internal link
  102. let ref = element
  103. .attr('href')
  104. .value()
  105. .replace(/\/links\/_[0-9]+\//, '');
  106. extractorUtils.removeAllAttributes(element);
  107. const anchorMatch = ref.match(/.html#(.+)$/);
  108. if (anchorMatch) {
  109. element.attr('data-doc-anchor-id', anchorMatch[1]);
  110. ref = ref.replace('#' + anchorMatch[1], '');
  111. }
  112. ref = '_' + ref.replace('.html', '');
  113. element.attr({
  114. class: 'hue-doc-internal-link',
  115. href: 'javascript:void(0);',
  116. 'data-doc-ref': ref
  117. });
  118. } else if (element.attr('href')) {
  119. // External link
  120. const href = element.attr('href').value();
  121. extractorUtils.removeAllAttributes(element);
  122. element.attr({ class: 'hue-doc-external-link', href: href, target: '_blank' });
  123. }
  124. break;
  125. case 'h1':
  126. element.text(element.text().replace(/LanguageManual\s(.+)/, '$1'));
  127. element.attr({ class: 'hue-doc-title-hive' });
  128. break;
  129. case 'table':
  130. case 'td':
  131. case 'th':
  132. case 'tr':
  133. element.attr({ class: 'hue-doc-' + element.name() + '-hive' });
  134. default:
  135. }
  136. element.childNodes().forEach(adaptElement);
  137. };
  138. epub.on('end', () => {
  139. const rootTopics = [];
  140. const lastTopicPerLevel = {};
  141. const promises = [];
  142. epub.flow.forEach(chapter => {
  143. promises.push(
  144. new Promise((resolve, reject) => {
  145. const topic = new Topic('/', chapter.id);
  146. topic.fragment = {
  147. title: {
  148. text: () => chapter.title.replace(/LanguageManual\s(.+)/, '$1')
  149. }
  150. };
  151. epub.getChapter(chapter.id, (error, text) => {
  152. try {
  153. const contents = libxml.parseHtmlFragment('<div>' + text + '</div>');
  154. topic.domXml = contents.root();
  155. adaptElement(topic.domXml);
  156. resolve();
  157. } catch (error) {
  158. reject();
  159. }
  160. });
  161. if (lastTopicPerLevel[chapter.level - 1]) {
  162. lastTopicPerLevel[chapter.level - 1].children.push(topic);
  163. }
  164. if (chapter.level === 0) {
  165. rootTopics.push(topic);
  166. }
  167. lastTopicPerLevel[chapter.level] = topic;
  168. })
  169. );
  170. });
  171. Promise.all(promises).then(() => {
  172. const rootTopic = rootTopics[0];
  173. rootTopic.children.forEach(childTopic => {
  174. rootTopics.push(childTopic);
  175. });
  176. rootTopic.children = [];
  177. jsonHandler
  178. .saveTopics(rootTopics, outputPath, mako, false)
  179. .then(() => {
  180. console.log('Done.');
  181. })
  182. .catch(() => {
  183. console.log('Fail.');
  184. });
  185. });
  186. });
  187. epub.parse();