// Licensed to Cloudera, Inc. under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. Cloudera, Inc. licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
let fs = require('fs');
let libxml = require('libxmljs');
let keyDefs = {};
let pathToXref = {};
let knownTitles = {};
let xrefs = {};
let conRefs = {};
let handleChildNodes = (x, path, body) => {
x.childNodes().forEach(y => {
handleElement(y, path, body);
});
};
let addStartElement = (x, path, body, elemName, classes) => {
let start = '<' + elemName;
if (x.attr('id')) {
let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value();
start += ' id="' + id + '"';
}
if (classes) {
start += ' class="' + classes + '"';
}
start += '>';
body.push(start);
};
let wrapHtmlElement = (x, path, body, elemName, classes) => {
addStartElement(x, path, body, elemName, classes);
handleChildNodes(x, path, body);
body.push('' + elemName + '>');
};
let handleElement = (x, path, body) => {
if (x.name() === 'text') {
if (x.text().trim()) {
body.push(x.text());
}
} else if (x.name() === 'xref') {
body.push({
xrefNode: x,
path: path
});
} else {
if (!x.childNodes().length) {
if (x.name() === 'keyword' && x.attr('keyref')) {
if (keyDefs[x.attr('keyref').value()]) {
body.push(keyDefs[x.attr('keyref').value()]);
} else {
body.push(x.attr('keyref').value())
}
} else if (x.attr('conref') && x.attr('conref').value().indexOf('impala_common.xml') !== -1) {
var id = x.attr('conref').value().replace(/^.*common\//, '');
if (conRefs[id]) {
handleElement(conRefs[id], path, body);
} else {
console.log('concept ref not found with id: ' + id);
}
}
} else {
switch (x.name()) {
case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; }
case 'p': {
wrapHtmlElement(x, path, body, 'p');
break;
}
case 'concept': {
if (!x.attr('audience') || x.attr('audience').value() !== 'hidden') {
wrapHtmlElement(x, path, body, 'div');
if (x.attr('id') && x.get('title')) {
var titleParts = [];
handleElement(x.get('title'), path, titleParts);
knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join('');
}
}
break;
}
case 'conbody': {
wrapHtmlElement(x, path, body, 'div');
if (x.parent().get('title')) {
var titleParts = [];
handleElement(x.parent().get('title'), path, titleParts);
knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join('');
}
break;
}
case 'codeph':
case 'cmdname':
case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; }
case 'codeblock': {
addStartElement(x, path, body, 'div', 'sql-docs-code-block');
var preChildren = [];
handleChildNodes(x, path, preChildren);
preChildren.forEach(function (child) {
if (typeof child === 'string') {
body.push(child.replace(/\n/g, '
'));
} else if (child.xrefNode) {
body.push(child);
} else {
console.log('Could not process codeblock child: ' + child.toString());
}
});
body.push('');
break;
}
case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; }
case 'varname':
case 'filepath':
case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; }
case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; }
case 'example': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-example'); break; }
case 'b':
case 'dl':
case 'dlentry':
case 'ol':
case 'dd':
case 'dt':
case 'q':
case 'i':
case 'sup':
case 'ul':
case 'li': { wrapHtmlElement(x, path, body, x.name()); break; }
case 'indexterm':
case 'metadata':
case 'fig':
case 'prolog':
case 'titlealts':
case 'uicontrol':
case 'table':
case 'navtitle': break;
default: console.log('Could not process element of type: ' + x.name() + ' in ' + path);
}
}
}
};
let parseDml = (path) => {
return new Promise((resolve) => {
fs.readFile(path, 'utf8', (err, data) => {
try {
let xmlDoc = libxml.parseXmlString(data);
let body = [];
var titleParts = [];
handleChildNodes(xmlDoc.get('//title'), path, titleParts);
if (xmlDoc.root().attr('id')) {
knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join('');
}
xmlDoc.get('//title').remove();
xmlDoc.childNodes().forEach(x => {
handleElement(x, path, body);
});
resolve({ title: titleParts.join(''), body: body});
} catch (err) {
console.log(path);
console.log(err);
}
});
});
};
let flattenBody = (body, prefix) => {
let bodyString = '';
body.forEach(function (bodyElement) {
if (typeof bodyElement === 'string') {
bodyString += bodyElement;
} else if (bodyElement.xrefNode) {
if (bodyElement.xrefNode.attr('href')) {
if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') {
bodyString += '' + bodyElement.xrefNode.text() + ''
} else {
let href = bodyElement.xrefNode.attr('href').value();
if (href.indexOf('#') === 0) {
href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href;
} else if (href.indexOf('topics/') !== -1) {
href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc.
} else {
href = 'topics/' + href;
}
var split = href.split('#');
var unknown = false;
let title = href;
if (knownTitles[href]) {
title = bodyElement.xrefNode.text() || knownTitles[href];
} else if (knownTitles[split[0]]) {
title = bodyElement.xrefNode.text() || knownTitles[split[0]];
} else if (bodyElement.xrefNode.text()) {
unknown = true;
title = bodyElement.xrefNode.text();
} else if (split[1]) {
unknown = true;
title = split[1].replace(/_/g, ' ');
} else {
unknown = true;
title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' ');
}
if (unknown) {
bodyString += '' + title + ''; // Unknown = not parsed reference as some docs are excluded
} else {
bodyString += '' + title + '';
}
}
}
}
});
return bodyString;
};
let stringifyTopic = (topic, prefix) => {
let result = prefix + '{\n' + prefix + ' id: \'' + topic.ref + '\',\n' + prefix + ' title: \'' + topic.title + '\',\n' + prefix + ' weight: 1,\n' + prefix + ' bodyMatch: ko.observable(),\n' + prefix + ' open: ko.observable(false),\n' + prefix +' titleMatch: ko.observable()';
if (topic.body.length) {
result += ',\n' + prefix + ' body: \'' + flattenBody(topic.body, prefix) + '\''
}
if (topic.children.length) {
result += ',\n' + prefix + ' children: [\n';
let stringifiedChildren = [];
topic.children.forEach(child => {
stringifiedChildren.push(stringifyTopic(child, prefix + ' '))
});
result += stringifiedChildren.join(',\n');
result += prefix + ']';
} else {
result += ',\n' + prefix + ' children: []\n';
}
result += prefix + '}';
return result;
};
class Topic {
constructor (ref, node, promises) {
this.ref = ref;
this.path = '../Impala/docs/' + ref;
this.children = [];
this.title = '';
this.body = [];
if (pathToXref[this.ref]) {
pathToXref[this.ref].parsed = true;
}
promises.push(parseDml(this.path).then(parseResult => {
this.title = parseResult.title;
this.body = parseResult.body;
}));
if (node.childNodes().length) {
node.childNodes().forEach(x => {
if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
this.children.push(new Topic(x.attr('href').value(), x, promises));
}
});
}
}
toJson() {
return JSON.stringify({
body: flattenBody(this.body, ''),
title: this.title
})
}
}
fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => {
if (err) {
console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)');
console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder');
return;
}
libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => {
if (x.name() === 'keydef' && x.attr('keys')) {
let valNode = x.get('topicmeta/keywords/keyword');
if (valNode) {
keyDefs[x.attr('keys').value()] = valNode.text();
} else if (x.attr('href')) {
xrefs[x.attr('keys')] = {
ref: x.attr('href').value(),
parsed: false,
external: x.attr('scope') && x.attr('scope').value() === 'external'
};
pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')];
}
}
});
fs.readFile('../Impala/docs/shared/impala_common.xml', 'utf-8', (err, commonRaw) => {
let handleCommonChildren = children => {
children.forEach(child => {
if (child.attr('id')) {
conRefs[child.attr('id').value()] = child;
}
if (child.childNodes().length) {
handleCommonChildren(child.childNodes());
}
})
};
handleCommonChildren(libxml.parseXmlString(commonRaw).get('//conbody').childNodes());
fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => {
let topics = [];
let promises = [];
libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => {
if (x.name() === 'topicref') {
topics.push(new Topic(x.attr('href').value(), x, promises));
}
});
let index = {};
let topLevel = [];
Promise.all(promises).then(() => {
let saveTopics = (topics, parent) => {
topics.forEach(topic => {
let entry = {
title: topic.title,
ref: topic.ref,
children: []
};
if (!parent) {
topLevel.push(entry)
} else {
parent.children.push(entry);
}
let fileName = topic.ref.replace('topics/', '').replace('.xml', '.json');
index[topic.ref] = fileName;
fs.writeFile('desktop/core/src/desktop/static/desktop/docs/impala/' + fileName, topic.toJson(), () => { });
saveTopics(topic.children, entry);
})
};
saveTopics(topics);
fs.readFile('tools/sql-docs/impala_doc_index.mako.template', 'utf-8', (err, contents) => {
let indexStrings = [];
Object.keys(index).forEach(key => {
indexStrings.push('\'' + key + '\':\'${ static(\'desktop/docs/impala/' + index[key] + '\') }\'')
});
contents = contents.replace('/* docIndex */', indexStrings.join(','));
let createTopicJs = (entry) => {
return '{title:\'' + entry.title +'\',ref:\'' + entry.ref + '\',children:[' + entry.children.map(createTopicJs).join(',') + ']}';
};
contents = contents.replace('/* topLevel */', topLevel.map(createTopicJs).join(','));
fs.writeFile('desktop/core/src/desktop/templates/impala_doc_index.mako', contents, () => {
console.log('desktop/core/src/desktop/templates/impala_doc_index.mako written.');
})
});
});
});
})
});