hiveExtractor.js 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. /* eslint-disable no-restricted-syntax */
  17. const program = require('commander');
  18. const EPub = require('epub');
  19. const Topic = require('./Topic');
  20. const libxml = require('libxmljs');
  21. const extractorUtils = require('./extractorUtils');
  22. const outputPath = './desktop/core/src/desktop/static/desktop/docs/hive/';
  23. const mako = './desktop/core/src/desktop/templates/sql_doc_index.mako';
  24. const jsonHandler = require('./jsonHandler');
  25. // Handle command line arguments
  26. program
  27. .version('1.0')
  28. .option('-e, --epub [path]', 'the path to the Hive epub file')
  29. .parse(process.argv);
  30. const epub = new EPub(program.epub);
  31. const convertToPre = (element, fragments) => {
  32. switch (element.name()) {
  33. case 'div':
  34. element.childNodes().forEach(node => {
  35. convertToPre(node, fragments);
  36. });
  37. break;
  38. case 'text':
  39. if (fragments.length === 1) {
  40. fragments.push(element.text().replace(/^\n/, ''));
  41. } else {
  42. fragments.push(element.text());
  43. }
  44. break;
  45. case 'code':
  46. if (element.attr('class') && element.attr('class').value().indexOf('value') !== -1) {
  47. fragments.push('<span class="hue-doc-varname">');
  48. element.childNodes().forEach(node => {
  49. convertToPre(node, fragments);
  50. });
  51. fragments.push('</span>');
  52. break;
  53. }
  54. default:
  55. element.childNodes().forEach(node => {
  56. convertToPre(node, fragments);
  57. });
  58. }
  59. };
  60. const adaptElement = element => {
  61. if (element.attr('class') && element.attr('class').value().indexOf('syntaxhighlighter') !== -1) {
  62. const fragments = ['<div class="hue-doc-codeblock">'];
  63. element.childNodes().forEach(childNode => {
  64. convertToPre(childNode, fragments);
  65. });
  66. fragments.push('</div>');
  67. const replacement = fragments.join('');
  68. element.replace(libxml.parseHtmlFragment(replacement).root());
  69. } else if (element.attr('class')) {
  70. if (element.attr('class').value().indexOf('admonition') !== -1) {
  71. element.attr({ class: 'hue-doc-note-hive' });
  72. } else {
  73. element.attr('class').remove();
  74. }
  75. }
  76. switch (element.name()) {
  77. case 'a':
  78. if (element.attr('href') && element.attr('href').value().indexOf('/links') == 0) {
  79. // Internal link
  80. let ref = element
  81. .attr('href')
  82. .value()
  83. .replace(/\/links\/_[0-9]+\//, '');
  84. extractorUtils.removeAllAttributes(element);
  85. const anchorMatch = ref.match(/.html#(.+)$/);
  86. if (anchorMatch) {
  87. element.attr('data-doc-anchor-id', anchorMatch[1]);
  88. ref = ref.replace('#' + anchorMatch[1], '');
  89. }
  90. ref = '_' + ref.replace('.html', '');
  91. element.attr({
  92. class: 'hue-doc-internal-link',
  93. href: 'javascript:void(0);',
  94. 'data-doc-ref': ref
  95. });
  96. } else if (element.attr('href')) {
  97. // External link
  98. const href = element.attr('href').value();
  99. extractorUtils.removeAllAttributes(element);
  100. element.attr({ class: 'hue-doc-external-link', href: href, target: '_blank' });
  101. }
  102. break;
  103. case 'h1':
  104. element.text(element.text().replace(/LanguageManual\s(.+)/, '$1'));
  105. element.attr({ class: 'hue-doc-title-hive' });
  106. break;
  107. case 'table':
  108. case 'td':
  109. case 'th':
  110. case 'tr':
  111. element.attr({ class: 'hue-doc-' + element.name() + '-hive' });
  112. default:
  113. }
  114. element.childNodes().forEach(adaptElement);
  115. };
  116. epub.on('end', () => {
  117. const rootTopics = [];
  118. const lastTopicPerLevel = {};
  119. const promises = [];
  120. epub.flow.forEach(chapter => {
  121. promises.push(
  122. new Promise((resolve, reject) => {
  123. const topic = new Topic('/', chapter.id);
  124. topic.fragment = {
  125. title: {
  126. text: () => chapter.title.replace(/LanguageManual\s(.+)/, '$1')
  127. }
  128. };
  129. epub.getChapter(chapter.id, (error, text) => {
  130. try {
  131. const contents = libxml.parseHtmlFragment('<div>' + text + '</div>');
  132. topic.domXml = contents.root();
  133. adaptElement(topic.domXml);
  134. resolve();
  135. } catch (error) {
  136. reject();
  137. }
  138. });
  139. if (lastTopicPerLevel[chapter.level - 1]) {
  140. lastTopicPerLevel[chapter.level - 1].children.push(topic);
  141. }
  142. if (chapter.level === 0) {
  143. rootTopics.push(topic);
  144. }
  145. lastTopicPerLevel[chapter.level] = topic;
  146. })
  147. );
  148. });
  149. Promise.all(promises).then(() => {
  150. const rootTopic = rootTopics[0];
  151. rootTopic.children.forEach(childTopic => {
  152. rootTopics.push(childTopic);
  153. });
  154. rootTopic.children = [];
  155. jsonHandler
  156. .saveTopics(rootTopics, outputPath, mako, false)
  157. .then(() => {
  158. console.log('Done.');
  159. })
  160. .catch(() => {
  161. console.log('Fail.');
  162. });
  163. });
  164. });
  165. epub.parse();
  166. /* eslint-enable no-restricted-syntax */