| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289 |
- // Licensed to Cloudera, Inc. under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. Cloudera, Inc. licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- let fs = require('fs');
- let libxml = require('libxmljs');
- let keyDefs = {};
- let pathToXref = {};
- let knownTitles = {};
- let xrefs = {};
- let handleChildNodes = (x, path, body) => {
- x.childNodes().forEach(y => {
- handleElement(y, path, body);
- });
- };
- let addStartElement = (x, path, body, elemName, classes) => {
- let start = '<' + elemName;
- if (x.attr('id')) {
- let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value();
- start += ' id="' + id + '"';
- }
- if (classes) {
- start += ' class="' + classes + '"';
- }
- start += '>';
- body.push(start);
- };
- let wrapHtmlElement = (x, path, body, elemName, classes) => {
- addStartElement(x, path, body, elemName, classes);
- handleChildNodes(x, path, body);
- body.push('</' + elemName + '> ');
- };
- let handleElement = (x, path, body) => {
- if (x.name() === 'text') {
- if (x.text().trim()) {
- body.push(x.text());
- }
- } else if (x.name() === 'xref') {
- body.push({
- xrefNode: x,
- path: path
- });
- } else {
- if (!x.childNodes().length) {
- if (x.name() === 'keyword' && x.attr('keyref')) {
- if (keyDefs[x.attr('keyref').value()]) {
- body.push(keyDefs[x.attr('keyref').value()]);
- } else {
- body.push(x.attr('keyref').value())
- }
- }
- } else {
- switch (x.name()) {
- case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; }
- case 'p': { wrapHtmlElement(x, path, body, 'p'); break; }
- case 'concept': {
- wrapHtmlElement(x, path, body, 'div');
- if (x.attr('id') && x.get('title')) {
- var titleParts = [];
- handleElement(x.get('title'), path, titleParts);
- knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join('');
- }
- break;
- }
- case 'conbody': {
- wrapHtmlElement(x, path, body, 'div');
- if (x.parent().get('title')) {
- var titleParts = [];
- handleElement(x.parent().get('title'), path, titleParts);
- knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join('');
- }
- break;
- }
- case 'codeph':
- case 'cmdname':
- case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; }
- case 'codeblock': {
- addStartElement(x, path, body, 'div', 'sql-docs-code-block');
- var preChildren = [];
- handleChildNodes(x, path, preChildren);
- preChildren.forEach(function (child) {
- body.push(child.replace(/^\s*/g, '').replace(/\n/g, '<br/>'));
- });
- body.push('</div>');
- break;
- }
- case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; }
- case 'varname':
- case 'filepath':
- case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; }
- case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; }
- case 'b': { wrapHtmlElement(x, path, body, 'b', 'sql-docs-variable'); break; }
- case 'q': { wrapHtmlElement(x, path, body, 'q', 'sql-docs-variable'); break; }
- case 'i': { wrapHtmlElement(x, path, body, 'i', 'sql-docs-variable'); break; }
- case 'sup': { wrapHtmlElement(x, path, body, 'sup', 'sql-docs-variable'); break; }
- case 'ul': { wrapHtmlElement(x, path, body, 'ul', 'sql-docs-variable'); break; }
- case 'li': { wrapHtmlElement(x, path, body, 'li', 'sql-docs-variable'); break; }
- case 'indexterm':
- case 'metadata':
- case 'fig':
- case 'prolog':
- case 'navtitle': break;
- default: console.log('Could not process element of type: ' + x.name() + ' in ' + path);
- }
- }
- }
- };
- let parseDml = (path) => {
- return new Promise((resolve) => {
- fs.readFile(path, 'utf8', (err, data) => {
- try {
- let xmlDoc = libxml.parseXmlString(data);
- let body = [];
- var titleParts = [];
- handleChildNodes(xmlDoc.get('//title'), path, titleParts);
- if (xmlDoc.root().attr('id')) {
- knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join('');
- }
- xmlDoc.get('//title').remove();
- xmlDoc.childNodes().forEach(x => {
- handleChildNodes(x, path, body);
- });
- resolve({ title: titleParts.join(''), body: body});
- } catch (err) {
- console.log(path);
- console.log(err);
- }
- });
- });
- };
- class Topic {
- constructor (ref, node, promises) {
- this.ref = ref;
- this.path = '../Impala/docs/' + ref;
- this.children = [];
- this.title = '';
- this.body = [];
- if (pathToXref[this.ref]) {
- pathToXref[this.ref].parsed = true;
- }
- promises.push(parseDml(this.path).then(parseResult => {
- this.title = parseResult.title;
- this.body = parseResult.body;
- }));
- if (node.childNodes().length) {
- node.childNodes().forEach(x => {
- if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
- this.children.push(new Topic(x.attr('href').value(), x, promises));
- }
- });
- }
- }
- }
- fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => {
- if (err) {
- console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)');
- console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder');
- return;
- }
- libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => {
- if (x.name() === 'keydef' && x.attr('keys')) {
- let valNode = x.get('topicmeta/keywords/keyword');
- if (valNode) {
- keyDefs[x.attr('keys').value()] = valNode.text();
- } else if (x.attr('href')) {
- xrefs[x.attr('keys')] = {
- ref: x.attr('href').value(),
- parsed: false,
- external: x.attr('scope') && x.attr('scope').value() === 'external'
- };
- pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')];
- }
- }
- });
- let stringifyTopic = (topic, prefix) => {
- let result = prefix + '{\n' + prefix + ' id: \'' + topic.ref + '\',\n' + prefix + ' title: \'' + topic.title + '\',\n' + prefix + ' weight: 1,\n' + prefix + ' bodyMatch: ko.observable(),\n' + prefix + ' open: ko.observable(false),\n' + prefix +' titleMatch: ko.observable()';
- if (topic.body.length) {
- var bodyString = '';
- topic.body.forEach(function (bodyElement) {
- if (typeof bodyElement === 'string') {
- bodyString += bodyElement;
- } else if (bodyElement.xrefNode) {
- if (bodyElement.xrefNode.attr('href')) {
- if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') {
- bodyString += '<a target="_blank" href="' + bodyElement.xrefNode.attr('href').value() + '">' + bodyElement.xrefNode.text() + '</a>'
- } else {
- let href = bodyElement.xrefNode.attr('href').value();
- if (href.indexOf('#') === 0) {
- href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href;
- } else if (href.indexOf('topics/') !== -1) {
- href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc.
- } else {
- href = 'topics/' + href;
- }
- var split = href.split('#');
- var unknown = false;
- let title = href;
- if (knownTitles[href]) {
- title = bodyElement.xrefNode.text() || knownTitles[href];
- } else if (knownTitles[split[0]]) {
- title = bodyElement.xrefNode.text() || knownTitles[split[0]];
- } else if (bodyElement.xrefNode.text()) {
- unknown = true;
- title = bodyElement.xrefNode.text();
- } else if (split[1]) {
- unknown = true;
- title = split[1].replace(/_/g, ' ');
- } else {
- unknown = true;
- title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' ');
- }
- if (unknown) {
- bodyString += '<span>"' + title + '"</span>'; // Unknown = not parsed reference as some docs are excluded
- } else {
- bodyString += '<a href="javascript: void(0);" class="lang-ref-link" data-target="' + href + '">' + title + '</a>';
- }
- }
- }
- }
- });
- result += ',\n' + prefix + ' body: \'' + bodyString.replace(/([^\\])\\([^\\])/g, '$1\\\\$2').replace(/'/g, '\\\'').replace(/\n/g, '\' + \n' + prefix + ' \'') + '\''
- }
- if (topic.children.length) {
- result += ',\n' + prefix + ' children: [\n';
- let stringifiedChildren = [];
- topic.children.forEach(child => {
- stringifiedChildren.push(stringifyTopic(child, prefix + ' '))
- });
- result += stringifiedChildren.join(',\n');
- result += prefix + ']';
- } else {
- result += ',\n' + prefix + ' children: []\n';
- }
- result += prefix + '}';
- return result;
- };
- fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => {
- let topics = [];
- let promises = [];
- libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => {
- if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
- topics.push(new Topic(x.attr('href').value(), x, promises));
- }
- });
- Promise.all(promises).then(() => {
- fs.readFile('tools/jison/license.txt', 'utf-8', (err, licenseHeader) => {
- let result = licenseHeader + '\n\n\n// NOTE: This is a generated file!\n// Run \'node tools/sql-docs/impalaExtractor.js\' to generate.\n\n\nvar impalaLangRefTopics = [\n';
- let stringifiedTopics = [];
- topics.forEach(topic => {
- stringifiedTopics.push(stringifyTopic(topic, ''));
- });
- result += stringifiedTopics.join(',\n');
- result += '\n];';
- fs.writeFile('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js', result, () => {
- console.log('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js updated!');
- })
- })
- });
- });
- });
|