// Licensed to Cloudera, Inc. under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. Cloudera, Inc. licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. let fs = require('fs'); let libxml = require('libxmljs'); let keyDefs = {}; let pathToXref = {}; let knownTitles = {}; let xrefs = {}; let handleChildNodes = (x, path, body) => { x.childNodes().forEach(y => { handleElement(y, path, body); }); }; let addStartElement = (x, path, body, elemName, classes) => { let start = '<' + elemName; if (x.attr('id')) { let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value(); start += ' id="' + id + '"'; } if (classes) { start += ' class="' + classes + '"'; } start += '>'; body.push(start); }; let wrapHtmlElement = (x, path, body, elemName, classes) => { addStartElement(x, path, body, elemName, classes); handleChildNodes(x, path, body); body.push(' '); }; let handleElement = (x, path, body) => { if (x.name() === 'text') { if (x.text().trim()) { body.push(x.text()); } } else if (x.name() === 'xref') { body.push({ xrefNode: x, path: path }); } else { if (!x.childNodes().length) { if (x.name() === 'keyword' && x.attr('keyref')) { if (keyDefs[x.attr('keyref').value()]) { body.push(keyDefs[x.attr('keyref').value()]); } else { body.push(x.attr('keyref').value()) } } } else { switch (x.name()) { case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; } case 'p': { wrapHtmlElement(x, path, body, 'p'); break; } case 'concept': { wrapHtmlElement(x, path, body, 'div'); if (x.attr('id') && x.get('title')) { var titleParts = []; handleElement(x.get('title'), path, titleParts); knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join(''); } break; } case 'conbody': { wrapHtmlElement(x, path, body, 'div'); if (x.parent().get('title')) { var titleParts = []; handleElement(x.parent().get('title'), path, titleParts); knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join(''); } break; } case 'codeph': case 'cmdname': case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; } case 'codeblock': { addStartElement(x, path, body, 'div', 'sql-docs-code-block'); var preChildren = []; handleChildNodes(x, path, preChildren); preChildren.forEach(function (child) { body.push(child.replace(/^\s*/g, '').replace(/\n/g, '
')); }); body.push(''); break; } case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; } case 'varname': case 'filepath': case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; } case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; } case 'b': { wrapHtmlElement(x, path, body, 'b', 'sql-docs-variable'); break; } case 'q': { wrapHtmlElement(x, path, body, 'q', 'sql-docs-variable'); break; } case 'i': { wrapHtmlElement(x, path, body, 'i', 'sql-docs-variable'); break; } case 'sup': { wrapHtmlElement(x, path, body, 'sup', 'sql-docs-variable'); break; } case 'ul': { wrapHtmlElement(x, path, body, 'ul', 'sql-docs-variable'); break; } case 'li': { wrapHtmlElement(x, path, body, 'li', 'sql-docs-variable'); break; } case 'indexterm': case 'metadata': case 'fig': case 'prolog': case 'navtitle': break; default: console.log('Could not process element of type: ' + x.name() + ' in ' + path); } } } }; let parseDml = (path) => { return new Promise((resolve) => { fs.readFile(path, 'utf8', (err, data) => { try { let xmlDoc = libxml.parseXmlString(data); let body = []; var titleParts = []; handleChildNodes(xmlDoc.get('//title'), path, titleParts); if (xmlDoc.root().attr('id')) { knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join(''); } xmlDoc.get('//title').remove(); xmlDoc.childNodes().forEach(x => { handleChildNodes(x, path, body); }); resolve({ title: titleParts.join(''), body: body}); } catch (err) { console.log(path); console.log(err); } }); }); }; class Topic { constructor (ref, node, promises) { this.ref = ref; this.path = '../Impala/docs/' + ref; this.children = []; this.title = ''; this.body = []; if (pathToXref[this.ref]) { pathToXref[this.ref].parsed = true; } promises.push(parseDml(this.path).then(parseResult => { this.title = parseResult.title; this.body = parseResult.body; })); if (node.childNodes().length) { node.childNodes().forEach(x => { if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') { this.children.push(new Topic(x.attr('href').value(), x, promises)); } }); } } } fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => { if (err) { console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)'); console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder'); return; } libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => { if (x.name() === 'keydef' && x.attr('keys')) { let valNode = x.get('topicmeta/keywords/keyword'); if (valNode) { keyDefs[x.attr('keys').value()] = valNode.text(); } else if (x.attr('href')) { xrefs[x.attr('keys')] = { ref: x.attr('href').value(), parsed: false, external: x.attr('scope') && x.attr('scope').value() === 'external' }; pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')]; } } }); let stringifyTopic = (topic, prefix) => { let result = prefix + '{\n' + prefix + ' id: \'' + topic.ref + '\',\n' + prefix + ' title: \'' + topic.title + '\',\n' + prefix + ' weight: 1,\n' + prefix + ' bodyMatch: ko.observable(),\n' + prefix + ' open: ko.observable(false),\n' + prefix +' titleMatch: ko.observable()'; if (topic.body.length) { var bodyString = ''; topic.body.forEach(function (bodyElement) { if (typeof bodyElement === 'string') { bodyString += bodyElement; } else if (bodyElement.xrefNode) { if (bodyElement.xrefNode.attr('href')) { if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') { bodyString += '' + bodyElement.xrefNode.text() + '' } else { let href = bodyElement.xrefNode.attr('href').value(); if (href.indexOf('#') === 0) { href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href; } else if (href.indexOf('topics/') !== -1) { href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc. } else { href = 'topics/' + href; } var split = href.split('#'); var unknown = false; let title = href; if (knownTitles[href]) { title = bodyElement.xrefNode.text() || knownTitles[href]; } else if (knownTitles[split[0]]) { title = bodyElement.xrefNode.text() || knownTitles[split[0]]; } else if (bodyElement.xrefNode.text()) { unknown = true; title = bodyElement.xrefNode.text(); } else if (split[1]) { unknown = true; title = split[1].replace(/_/g, ' '); } else { unknown = true; title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' '); } if (unknown) { bodyString += '"' + title + '"'; // Unknown = not parsed reference as some docs are excluded } else { bodyString += '' + title + ''; } } } } }); result += ',\n' + prefix + ' body: \'' + bodyString.replace(/([^\\])\\([^\\])/g, '$1\\\\$2').replace(/'/g, '\\\'').replace(/\n/g, '\' + \n' + prefix + ' \'') + '\'' } if (topic.children.length) { result += ',\n' + prefix + ' children: [\n'; let stringifiedChildren = []; topic.children.forEach(child => { stringifiedChildren.push(stringifyTopic(child, prefix + ' ')) }); result += stringifiedChildren.join(',\n'); result += prefix + ']'; } else { result += ',\n' + prefix + ' children: []\n'; } result += prefix + '}'; return result; }; fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => { let topics = []; let promises = []; libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => { if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') { topics.push(new Topic(x.attr('href').value(), x, promises)); } }); Promise.all(promises).then(() => { fs.readFile('tools/jison/license.txt', 'utf-8', (err, licenseHeader) => { let result = licenseHeader + '\n\n\n// NOTE: This is a generated file!\n// Run \'node tools/sql-docs/impalaExtractor.js\' to generate.\n\n\nvar impalaLangRefTopics = [\n'; let stringifiedTopics = []; topics.forEach(topic => { stringifiedTopics.push(stringifyTopic(topic, '')); }); result += stringifiedTopics.join(',\n'); result += '\n];'; fs.writeFile('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js', result, () => { console.log('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js updated!'); }) }) }); }); });