| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372 |
- // Licensed to Cloudera, Inc. under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. Cloudera, Inc. licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- let fs = require('fs');
- let libxml = require('libxmljs');
- let keyDefs = {};
- let pathToXref = {};
- let knownTitles = {};
- let xrefs = {};
- let conRefs = {};
- let handleChildNodes = (x, path, body) => {
- x.childNodes().forEach(y => {
- handleElement(y, path, body);
- });
- };
- let addStartElement = (x, path, body, elemName, classes) => {
- let start = '<' + elemName;
- if (x.attr('id')) {
- let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value();
- start += ' id="' + id + '"';
- }
- if (classes) {
- start += ' class="' + classes + '"';
- }
- start += '>';
- body.push(start);
- };
- let wrapHtmlElement = (x, path, body, elemName, classes) => {
- addStartElement(x, path, body, elemName, classes);
- handleChildNodes(x, path, body);
- body.push('</' + elemName + '>');
- };
- let handleElement = (x, path, body) => {
- if (x.name() === 'text') {
- if (x.text().trim()) {
- body.push(x.text());
- }
- } else if (x.name() === 'xref') {
- body.push({
- xrefNode: x,
- path: path
- });
- } else {
- if (!x.childNodes().length) {
- if (x.name() === 'keyword' && x.attr('keyref')) {
- if (keyDefs[x.attr('keyref').value()]) {
- body.push(keyDefs[x.attr('keyref').value()]);
- } else {
- body.push(x.attr('keyref').value())
- }
- } else if (x.attr('conref') && x.attr('conref').value().indexOf('impala_common.xml') !== -1) {
- var id = x.attr('conref').value().replace(/^.*common\//, '');
- if (conRefs[id]) {
- handleElement(conRefs[id], path, body);
- } else {
- console.log('concept ref not found with id: ' + id);
- }
- }
- } else {
- switch (x.name()) {
- case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; }
- case 'p': {
- wrapHtmlElement(x, path, body, 'p');
- break;
- }
- case 'concept': {
- if (!x.attr('audience') || x.attr('audience').value() !== 'hidden') {
- wrapHtmlElement(x, path, body, 'div');
- if (x.attr('id') && x.get('title')) {
- var titleParts = [];
- handleElement(x.get('title'), path, titleParts);
- knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join('');
- }
- }
- break;
- }
- case 'conbody': {
- wrapHtmlElement(x, path, body, 'div');
- if (x.parent().get('title')) {
- var titleParts = [];
- handleElement(x.parent().get('title'), path, titleParts);
- knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join('');
- }
- break;
- }
- case 'codeph':
- case 'cmdname':
- case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; }
- case 'codeblock': {
- addStartElement(x, path, body, 'div', 'sql-docs-code-block');
- var preChildren = [];
- handleChildNodes(x, path, preChildren);
- preChildren.forEach(function (child) {
- if (typeof child === 'string') {
- body.push(child.replace(/\n/g, '<br/>'));
- } else if (child.xrefNode) {
- body.push(child);
- } else {
- console.log('Could not process codeblock child: ' + child.toString());
- }
- });
- body.push('</div>');
- break;
- }
- case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; }
- case 'varname':
- case 'filepath':
- case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; }
- case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; }
- case 'example': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-example'); break; }
- case 'b':
- case 'dl':
- case 'dlentry':
- case 'ol':
- case 'dd':
- case 'dt':
- case 'q':
- case 'i':
- case 'sup':
- case 'ul':
- case 'li': { wrapHtmlElement(x, path, body, x.name()); break; }
- case 'indexterm':
- case 'metadata':
- case 'fig':
- case 'prolog':
- case 'titlealts':
- case 'uicontrol':
- case 'table':
- case 'navtitle': break;
- default: console.log('Could not process element of type: ' + x.name() + ' in ' + path);
- }
- }
- }
- };
- let parseDml = (path) => {
- return new Promise((resolve) => {
- fs.readFile(path, 'utf8', (err, data) => {
- try {
- let xmlDoc = libxml.parseXmlString(data);
- let body = [];
- var titleParts = [];
- handleChildNodes(xmlDoc.get('//title'), path, titleParts);
- if (xmlDoc.root().attr('id')) {
- knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join('');
- }
- xmlDoc.get('//title').remove();
- xmlDoc.childNodes().forEach(x => {
- handleElement(x, path, body);
- });
- resolve({ title: titleParts.join(''), body: body});
- } catch (err) {
- console.log(path);
- console.log(err);
- }
- });
- });
- };
- let flattenBody = (body, prefix) => {
- let bodyString = '';
- body.forEach(function (bodyElement) {
- if (typeof bodyElement === 'string') {
- bodyString += bodyElement;
- } else if (bodyElement.xrefNode) {
- if (bodyElement.xrefNode.attr('href')) {
- if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') {
- bodyString += '<a target="_blank" href="' + bodyElement.xrefNode.attr('href').value() + '">' + bodyElement.xrefNode.text() + '</a>'
- } else {
- let href = bodyElement.xrefNode.attr('href').value();
- if (href.indexOf('#') === 0) {
- href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href;
- } else if (href.indexOf('topics/') !== -1) {
- href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc.
- } else {
- href = 'topics/' + href;
- }
- var split = href.split('#');
- var unknown = false;
- let title = href;
- if (knownTitles[href]) {
- title = bodyElement.xrefNode.text() || knownTitles[href];
- } else if (knownTitles[split[0]]) {
- title = bodyElement.xrefNode.text() || knownTitles[split[0]];
- } else if (bodyElement.xrefNode.text()) {
- unknown = true;
- title = bodyElement.xrefNode.text();
- } else if (split[1]) {
- unknown = true;
- title = split[1].replace(/_/g, ' ');
- } else {
- unknown = true;
- title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' ');
- }
- if (unknown) {
- bodyString += '<span>' + title + '</span>'; // Unknown = not parsed reference as some docs are excluded
- } else {
- bodyString += '<a href="javascript: void(0);" class="lang-ref-link" data-target="' + href + '">' + title + '</a>';
- }
- }
- }
- }
- });
- return bodyString;
- };
- let stringifyTopic = (topic, prefix) => {
- let result = prefix + '{\n' + prefix + ' id: \'' + topic.ref + '\',\n' + prefix + ' title: \'' + topic.title + '\',\n' + prefix + ' weight: 1,\n' + prefix + ' bodyMatch: ko.observable(),\n' + prefix + ' open: ko.observable(false),\n' + prefix +' titleMatch: ko.observable()';
- if (topic.body.length) {
- result += ',\n' + prefix + ' body: \'' + flattenBody(topic.body, prefix) + '\''
- }
- if (topic.children.length) {
- result += ',\n' + prefix + ' children: [\n';
- let stringifiedChildren = [];
- topic.children.forEach(child => {
- stringifiedChildren.push(stringifyTopic(child, prefix + ' '))
- });
- result += stringifiedChildren.join(',\n');
- result += prefix + ']';
- } else {
- result += ',\n' + prefix + ' children: []\n';
- }
- result += prefix + '}';
- return result;
- };
- class Topic {
- constructor (ref, node, promises) {
- this.ref = ref;
- this.path = '../Impala/docs/' + ref;
- this.children = [];
- this.title = '';
- this.body = [];
- if (pathToXref[this.ref]) {
- pathToXref[this.ref].parsed = true;
- }
- promises.push(parseDml(this.path).then(parseResult => {
- this.title = parseResult.title;
- this.body = parseResult.body;
- }));
- if (node.childNodes().length) {
- node.childNodes().forEach(x => {
- if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
- this.children.push(new Topic(x.attr('href').value(), x, promises));
- }
- });
- }
- }
- toJson() {
- return JSON.stringify({
- body: flattenBody(this.body, ''),
- title: this.title
- })
- }
- }
- fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => {
- if (err) {
- console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)');
- console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder');
- return;
- }
- libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => {
- if (x.name() === 'keydef' && x.attr('keys')) {
- let valNode = x.get('topicmeta/keywords/keyword');
- if (valNode) {
- keyDefs[x.attr('keys').value()] = valNode.text();
- } else if (x.attr('href')) {
- xrefs[x.attr('keys')] = {
- ref: x.attr('href').value(),
- parsed: false,
- external: x.attr('scope') && x.attr('scope').value() === 'external'
- };
- pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')];
- }
- }
- });
- fs.readFile('../Impala/docs/shared/impala_common.xml', 'utf-8', (err, commonRaw) => {
- let handleCommonChildren = children => {
- children.forEach(child => {
- if (child.attr('id')) {
- conRefs[child.attr('id').value()] = child;
- }
- if (child.childNodes().length) {
- handleCommonChildren(child.childNodes());
- }
- })
- };
- handleCommonChildren(libxml.parseXmlString(commonRaw).get('//conbody').childNodes());
- fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => {
- let topics = [];
- let promises = [];
- libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => {
- if (x.name() === 'topicref') {
- topics.push(new Topic(x.attr('href').value(), x, promises));
- }
- });
- let index = {};
- let topLevel = [];
- Promise.all(promises).then(() => {
- let saveTopics = (topics, parent) => {
- topics.forEach(topic => {
- let entry = {
- title: topic.title,
- ref: topic.ref,
- children: []
- };
- if (!parent) {
- topLevel.push(entry)
- } else {
- parent.children.push(entry);
- }
- let fileName = topic.ref.replace('topics/', '').replace('.xml', '.json');
- index[topic.ref] = fileName;
- fs.writeFile('desktop/core/src/desktop/static/desktop/docs/impala/' + fileName, topic.toJson(), () => { });
- saveTopics(topic.children, entry);
- })
- };
- saveTopics(topics);
- fs.readFile('tools/sql-docs/impala_doc_index.mako.template', 'utf-8', (err, contents) => {
- let indexStrings = [];
- Object.keys(index).forEach(key => {
- indexStrings.push('\'' + key + '\':\'${ static(\'desktop/docs/impala/' + index[key] + '\') }\'')
- });
- contents = contents.replace('/* docIndex */', indexStrings.join(','));
- let createTopicJs = (entry) => {
- return '{title:\'' + entry.title +'\',ref:\'' + entry.ref + '\',children:[' + entry.children.map(createTopicJs).join(',') + ']}';
- };
- contents = contents.replace('/* topLevel */', topLevel.map(createTopicJs).join(','));
- fs.writeFile('desktop/core/src/desktop/templates/impala_doc_index.mako', contents, () => {
- console.log('desktop/core/src/desktop/templates/impala_doc_index.mako written.');
- })
- });
- });
- });
- })
- });
|