Browse Source

HUE-6869 [editor] Add Impala language reference to the right assist

The language reference is generated by running "node tools/sql-docs/impalaExtractor.js" in the hue folder.

(cherry picked from commit 6e4ae824af4bf54a877b21785cb4d42161f96827)
Johan Ahlen 7 years ago
parent
commit
8b1e8e3b61

File diff suppressed because it is too large
+ 0 - 0
desktop/core/src/desktop/static/desktop/css/hue-embedded.css


File diff suppressed because it is too large
+ 0 - 0
desktop/core/src/desktop/static/desktop/css/hue.css


File diff suppressed because it is too large
+ 0 - 0
desktop/core/src/desktop/static/desktop/css/hue3-extra.css


File diff suppressed because it is too large
+ 1166 - 0
desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js


+ 22 - 1
desktop/core/src/desktop/static/desktop/less/hue-assist.less

@@ -634,9 +634,30 @@
 .assist-function-details {
   border-top: 2px solid @cui-default-border-color;
   margin-top: 5px;
-  margin-right:10px;
+  margin-right: 10px;
   padding: 10px;
   white-space: normal;
+
+  .sql-docs-inline-code {
+    font-family: @font-family-monospace;
+  }
+
+  .sql-docs-code-block {
+    display: block;
+    font-family: @font-family-monospace;
+    white-space: pre;
+    margin-bottom: 10px;
+    color: @cui-gray-700;
+  }
+
+  .sql-docs-variable {
+    font-family: @font-family-monospace;
+    color: @cui-steel;
+  }
+
+  .sql-docs-note {
+
+  }
 }
 
 .assist-function-signature {

+ 106 - 0
desktop/core/src/desktop/templates/assist.mako

@@ -2222,6 +2222,55 @@ from desktop.views import _ko
     })();
   </script>
 
+  <script type="text/html" id="language-reference-panel-template">
+    <div class="assist-inner-panel">
+      <div class="assist-flex-panel">
+        <div class="assist-flex-search">
+          <div class="assist-filter">
+            <input class="clearable" type="text" placeholder="Filter..." data-bind="clearable: query, value: query, valueUpdate: 'afterkeydown'">
+          </div>
+        </div>
+        <div data-bind="css: { 'assist-flex-fill': !selectedTopic(), 'assist-flex-half': selectedTopic() }">
+          <!-- ko ifnot: query -->
+          <ul class="assist-functions" data-bind="foreach: availableTopics">
+            <li>
+              <a class="assist-field-link" href="javascript: void(0);" data-bind="css: { 'blue': $parent.selectedTopic() === $data }, click: function () { $parent.selectedTopic($data); }, toggle: open, text: title"></a>
+              <!-- ko if: open -->
+              <ul class="assist-functions" data-bind="foreach: children">
+                <li>
+                  <a class="assist-field-link" href="javascript: void(0);" data-bind="css: { 'blue': $parents[1].selectedTopic() === $data }, click: function () { $parents[1].selectedTopic($data); }, toggle: open, text: title"></a>
+                </li>
+              </ul>
+              <!-- /ko -->
+            </li>
+          </ul>
+          <!-- /ko -->
+          <!-- ko if: query -->
+          <!-- ko if: filteredTopics().length > 0 -->
+          <ul class="assist-functions" data-bind="foreach: filteredTopics">
+            <li>
+              <a class="assist-field-link" href="javascript: void(0);" data-bind="css: { 'blue': $parent.selectedTopic() === $data }, click: function () { $parent.selectedTopic($data); }, html: titleMatch() || title"></a>
+            </li>
+          </ul>
+          <!-- /ko -->
+          <!-- ko if: filteredTopics().length === 0 -->
+          <ul class="assist-functions">
+            <li class="assist-no-entries">${ _('No matches found. ') }</li>
+          </ul>
+          <!-- /ko -->
+          <!-- /ko -->
+        </div>
+        <!-- ko if: selectedTopic -->
+        <div class="assist-flex-half assist-function-details" data-bind="with: selectedTopic">
+          <div class="assist-panel-close"><button class="close" data-bind="click: function() { $parent.selectedTopic(undefined); }">&times;</button></div>
+          <div class="assist-function-signature blue" data-bind="html: titleMatch() || title"></div>
+          <div data-bind="html: bodyMatch() || body"></div>
+        </div>
+        <!-- /ko -->
+      </div>
+    </div>
+  </script>
+
   <script type="text/html" id="functions-panel-template">
     <div class="assist-inner-panel">
       <div class="assist-flex-panel">
@@ -2278,6 +2327,56 @@ from desktop.views import _ko
 
   <script type="text/javascript">
     (function () {
+      function LanguageReferencePanel (params, element) {
+        var self = this;
+        self.availableTopics = impalaLangRefTopics;
+        self.selectedTopic = ko.observable();
+
+        self.query = ko.observable();
+        self.filteredTopics = ko.pureComputed(function () {
+          var lowerCaseQuery = self.query().toLowerCase();
+          var replaceRegexp = new RegExp('(' + lowerCaseQuery + ')', 'i');
+          var flattenedTopics = [];
+
+          var findInside = function (topic) {
+            if (topic.title.toLowerCase().indexOf(lowerCaseQuery) === 0) {
+              topic.weight = 1;
+              topic.titleMatch(topic.title.replace(replaceRegexp, '<b>$1</b>'));
+              topic.bodyMatch(undefined);
+              flattenedTopics.push(topic);
+            } else if (topic.body && topic.body.toLowerCase().indexOf(lowerCaseQuery) !== -1) {
+              topic.weight = 0;
+              topic.titleMatch(undefined);
+              topic.bodyMatch(topic.body.replace(replaceRegexp, '<b>$1</b>'));
+              flattenedTopics.push(topic);
+            } else {
+              topic.titleMatch(undefined);
+              topic.bodyMatch(undefined);
+            }
+            topic.children.forEach(findInside);
+          };
+
+          self.availableTopics.forEach(findInside);
+
+          flattenedTopics.sort(function (a, b) {
+            if (a.weight !== b.weight) {
+              return b.weight - a.weight;
+            }
+            return a.title.localeCompare(b.title);
+          });
+          return flattenedTopics;
+        })
+      }
+
+      ko.components.register('language-reference-panel', {
+        viewModel: {
+          createViewModel: function(params, componentInfo) {
+            return new LanguageReferencePanel(params, componentInfo.element)
+          }
+        },
+        template: { element: 'language-reference-panel-template' }
+      });
+
       function FunctionsPanel(params) {
         var self = this;
         self.categories = {};
@@ -3317,6 +3416,7 @@ from desktop.views import _ko
       <ul class="right-panel-tabs nav nav-pills">
         <li data-bind="css: { 'active' : activeTab() === 'editorAssistant' }, visible: editorAssistantTabAvailable" style="display:none;"><a href="javascript: void(0);" data-bind="click: function() { lastActiveTabEditor('editorAssistant'); activeTab('editorAssistant'); }">${ _('Assistant') }</a></li>
         <li data-bind="css: { 'active' : activeTab() === 'functions' }, visible: functionsTabAvailable" style="display:none;"><a href="javascript: void(0);" data-bind="click: function() { lastActiveTabEditor('functions'); activeTab('functions'); }">${ _('Functions') }</a></li>
+        <li data-bind="css: { 'active' : activeTab() === 'syntx' }, visible: langRefTabAvailable" style="display:none;"><a href="javascript: void(0);" data-bind="click: function() { lastActiveTabEditor('langRef'); activeTab('langRef'); }">${ _('Reference') }</a></li>
         <li data-bind="css: { 'active' : activeTab() === 'schedules' }, visible: schedulesTabAvailable" style="display:none;"><a href="javascript: void(0);" data-bind="click: function() { lastActiveTabEditor('schedules'); activeTab('schedules'); }">${ _('Schedule') }</a></li>
         <li data-bind="css: { 'active' : activeTab() === 'dashboardAssistant' }, visible: dashboardAssistantTabAvailable" style="display:none;"><a href="javascript: void(0);" data-bind="click: function() { lastActiveTabDashboard('dashboardAssistant'); activeTab('dashboardAssistant'); }">${ _('Assistant') }</a></li>
       </ul>
@@ -3330,6 +3430,10 @@ from desktop.views import _ko
         <div data-bind="component: { name: 'functions-panel' }, visible: activeTab() === 'functions'"></div>
         <!-- /ko -->
 
+        <!-- ko if: langRefTabAvailable -->
+        <div data-bind="component: { name: 'language-reference-panel' }, visible: activeTab() === 'langRef'"></div>
+        <!-- /ko -->
+
         <!-- ko if: dashboardAssistantTabAvailable -->
         <div data-bind="component: { name: 'dashboard-assistant-panel' }, visible: activeTab() === 'dashboardAssistant'"></div>
         <!-- /ko -->
@@ -3358,6 +3462,7 @@ from desktop.views import _ko
         self.editorAssistantTabAvailable = ko.observable(false);
         self.dashboardAssistantTabAvailable = ko.observable(false);
         self.functionsTabAvailable = ko.observable(false);
+        self.langRefTabAvailable = ko.observable(false);
         self.schedulesTabAvailable = ko.observable(false);
 
         var apiHelper = ApiHelper.getInstance();
@@ -3399,6 +3504,7 @@ from desktop.views import _ko
 
         var updateContentsForType = function (type) {
           self.functionsTabAvailable(type === 'hive' || type === 'impala' || type === 'pig');
+          self.langRefTabAvailable(type === 'impala');
           self.editorAssistantTabAvailable((!window.IS_EMBEDDED || window.EMBEDDED_ASSISTANT_ENABLED) && (type === 'hive' || type === 'impala'));
           self.dashboardAssistantTabAvailable(type === 'dashboard');
           self.schedulesTabAvailable(false);

+ 1 - 0
desktop/core/src/desktop/templates/hue.mako

@@ -512,6 +512,7 @@ ${ commonshare() | n,unicode }
 <script src="${ static('desktop/js/jquery.scrollup.js') }"></script>
 <script src="${ static('desktop/js/jquery.huedatatable.js') }"></script>
 <script src="${ static('desktop/js/sqlFunctions.js') }"></script>
+<script src="${ static('desktop/js/sqlImpalaLangRef.js') }"></script>
 <script src="${ static('desktop/ext/js/selectize.min.js') }"></script>
 <script src="${ static('desktop/js/ko.selectize.js') }"></script>
 <script src="${ static('desktop/js/ace/ace.js') }"></script>

File diff suppressed because it is too large
+ 472 - 80
package-lock.json


+ 3 - 1
package.json

@@ -25,9 +25,11 @@
     "knockout": "3.4.0",
     "knockout.mapping": "2.4.3",
     "less": "2.6.1",
+    "libxmljs": "^0.19.0",
     "lodash": "3.10.1",
+    "page": "^1.7.1",
     "pubsub.js": "1.4.3",
-    "page": "^1.7.1"
+    "xml2js": "^0.4.19"
   },
   "devDependencies": {
     "babel-core": "6.22.1",

+ 289 - 0
tools/sql-docs/impalaExtractor.js

@@ -0,0 +1,289 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+let fs = require('fs');
+let libxml = require('libxmljs');
+
+let keyDefs = {};
+let pathToXref = {};
+let knownTitles = {};
+let xrefs = {};
+
+let handleChildNodes = (x, path, body) => {
+  x.childNodes().forEach(y => {
+    handleElement(y, path, body);
+  });
+};
+
+let addStartElement = (x, path, body, elemName, classes) => {
+  let start = '<' + elemName;
+  if (x.attr('id')) {
+    let id = path.substring(path.lastIndexOf('/' + 1)) + '_' + x.attr('id').value();
+    start += ' id="' + id + '"';
+  }
+  if (classes) {
+    start += ' class="' + classes + '"';
+  }
+  start += '>';
+  body.push(start);
+};
+
+let wrapHtmlElement = (x, path, body, elemName, classes) => {
+  addStartElement(x, path, body, elemName, classes);
+  handleChildNodes(x, path, body);
+  body.push('</' + elemName + '> ');
+};
+
+let handleElement = (x, path, body) => {
+  if (x.name() === 'text') {
+    if (x.text().trim()) {
+      body.push(x.text());
+    }
+  } else if (x.name() === 'xref') {
+    body.push({
+      xrefNode: x,
+      path: path
+    });
+  } else {
+    if (!x.childNodes().length) {
+      if (x.name() === 'keyword' && x.attr('keyref')) {
+        if (keyDefs[x.attr('keyref').value()]) {
+          body.push(keyDefs[x.attr('keyref').value()]);
+        } else {
+          body.push(x.attr('keyref').value())
+        }
+      }
+    } else {
+      switch (x.name()) {
+        case 'title': { wrapHtmlElement(x, path, body, 'h4'); break; }
+        case 'p': { wrapHtmlElement(x, path, body, 'p'); break; }
+        case 'concept': {
+          wrapHtmlElement(x, path, body, 'div');
+          if (x.attr('id') && x.get('title')) {
+            var titleParts = [];
+            handleElement(x.get('title'), path, titleParts);
+            knownTitles[path.substring(path.indexOf('topics/')) + '#' + x.attr('id').value()] = titleParts.join('');
+          }
+          break;
+        }
+        case 'conbody': {
+          wrapHtmlElement(x, path, body, 'div');
+          if (x.parent().get('title')) {
+            var titleParts = [];
+            handleElement(x.parent().get('title'), path, titleParts);
+            knownTitles[path.substring(path.indexOf('topics/'))] = titleParts.join('');
+          }
+          break;
+        }
+        case 'codeph':
+        case 'cmdname':
+        case 'ph': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-inline-code'); break; }
+        case 'codeblock': {
+          addStartElement(x, path, body, 'div', 'sql-docs-code-block');
+          var preChildren = [];
+          handleChildNodes(x, path, preChildren);
+          preChildren.forEach(function (child) {
+            body.push(child.replace(/^\s*/g, '').replace(/\n/g, '<br/>'));
+          });
+          body.push('</div>');
+          break;
+        }
+        case 'keyword': { wrapHtmlElement(x, path, body, 'span'); break; }
+        case 'varname':
+        case 'filepath':
+        case 'term': { wrapHtmlElement(x, path, body, 'span', 'sql-docs-variable'); break; }
+        case 'note': { wrapHtmlElement(x, path, body, 'div', 'sql-docs-note'); break; }
+        case 'b': { wrapHtmlElement(x, path, body, 'b', 'sql-docs-variable'); break; }
+        case 'q':  { wrapHtmlElement(x, path, body, 'q', 'sql-docs-variable'); break; }
+        case 'i': { wrapHtmlElement(x, path, body, 'i', 'sql-docs-variable'); break; }
+        case 'sup': { wrapHtmlElement(x, path, body, 'sup', 'sql-docs-variable'); break; }
+        case 'ul': { wrapHtmlElement(x, path, body, 'ul', 'sql-docs-variable'); break; }
+        case 'li': { wrapHtmlElement(x, path, body, 'li', 'sql-docs-variable'); break; }
+        case 'indexterm':
+        case 'metadata':
+        case 'fig':
+        case 'prolog':
+        case 'navtitle': break;
+        default: console.log('Could not process element of type: ' + x.name() + ' in ' + path);
+      }
+    }
+  }
+};
+
+let parseDml = (path) => {
+  return new Promise((resolve) => {
+    fs.readFile(path, 'utf8', (err, data) => {
+      try {
+        let xmlDoc = libxml.parseXmlString(data);
+        let body = [];
+        var titleParts = [];
+        handleChildNodes(xmlDoc.get('//title'), path, titleParts);
+        if (xmlDoc.root().attr('id')) {
+          knownTitles[path.substring(path.indexOf('topics/')) + '#' + xmlDoc.root().attr('id').value()] = titleParts.join('');
+        }
+        xmlDoc.get('//title').remove();
+        xmlDoc.childNodes().forEach(x => {
+          handleChildNodes(x, path, body);
+        });
+        resolve({ title: titleParts.join(''), body: body});
+      } catch (err) {
+        console.log(path);
+        console.log(err);
+      }
+    });
+  });
+};
+
+class Topic {
+  constructor (ref, node, promises) {
+    this.ref = ref;
+    this.path = '../Impala/docs/' + ref;
+    this.children = [];
+    this.title = '';
+    this.body = [];
+    if (pathToXref[this.ref]) {
+      pathToXref[this.ref].parsed = true;
+    }
+    promises.push(parseDml(this.path).then(parseResult => {
+      this.title = parseResult.title;
+      this.body = parseResult.body;
+    }));
+
+    if (node.childNodes().length) {
+      node.childNodes().forEach(x => {
+        if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
+          this.children.push(new Topic(x.attr('href').value(), x, promises));
+        }
+      });
+    }
+  }
+}
+
+fs.readFile('../Impala/docs/impala_keydefs.ditamap', 'utf8', (err, keyDefRaw) => {
+  if (err) {
+    console.log('Could not find the Impala docs! (../Impala/docs/impala_keydefs.ditamap)');
+    console.log('Make sure you have Impala checked out in an "Impala" folder next to the hue folder');
+    return;
+  }
+
+  libxml.parseXmlString(keyDefRaw).get('//map').childNodes().forEach(x => {
+    if (x.name() === 'keydef' && x.attr('keys')) {
+      let valNode = x.get('topicmeta/keywords/keyword');
+      if (valNode) {
+        keyDefs[x.attr('keys').value()] = valNode.text();
+      } else if (x.attr('href')) {
+        xrefs[x.attr('keys')] = {
+          ref: x.attr('href').value(),
+          parsed: false,
+          external: x.attr('scope') && x.attr('scope').value() === 'external'
+        };
+        pathToXref[x.attr('href').value()] = xrefs[x.attr('keys')];
+      }
+    }
+  });
+
+  let stringifyTopic = (topic, prefix) => {
+    let result = prefix + '{\n' + prefix + '  id: \'' + topic.ref + '\',\n' + prefix + '  title: \'' + topic.title + '\',\n' + prefix + '  weight: 1,\n' + prefix + '  bodyMatch: ko.observable(),\n' + prefix + '  open: ko.observable(false),\n' + prefix +'  titleMatch: ko.observable()';
+
+    if (topic.body.length) {
+      var bodyString = '';
+      topic.body.forEach(function (bodyElement) {
+        if (typeof bodyElement === 'string') {
+          bodyString += bodyElement;
+        } else if (bodyElement.xrefNode) {
+          if (bodyElement.xrefNode.attr('href')) {
+            if (bodyElement.xrefNode.attr('scope') && bodyElement.xrefNode.attr('scope').value() === 'external') {
+              bodyString += '<a target="_blank" href="' + bodyElement.xrefNode.attr('href').value() + '">' + bodyElement.xrefNode.text() + '</a>'
+            } else {
+              let href = bodyElement.xrefNode.attr('href').value();
+              if (href.indexOf('#') === 0) {
+                href = bodyElement.path.substring(bodyElement.path.indexOf('topics/')) + href;
+              } else if (href.indexOf('topics/') !== -1) {
+                href = href.substring(href.indexOf('topics')); // clean up [..]/topic/ etc.
+              } else {
+                href = 'topics/' + href;
+              }
+              var split = href.split('#');
+
+              var unknown = false;
+              let title = href;
+              if (knownTitles[href]) {
+                title = bodyElement.xrefNode.text() || knownTitles[href];
+              } else if (knownTitles[split[0]]) {
+                title = bodyElement.xrefNode.text() || knownTitles[split[0]];
+              } else if (bodyElement.xrefNode.text()) {
+                unknown = true;
+                title = bodyElement.xrefNode.text();
+              } else if (split[1]) {
+                unknown = true;
+                title = split[1].replace(/_/g, ' ');
+              } else {
+                unknown = true;
+                title = href.replace('topics/', '').replace('.xml', '').replace(/_/g, ' ');
+              }
+
+              if (unknown) {
+                bodyString += '<span>"' + title + '"</span>'; // Unknown = not parsed reference as some docs are excluded
+              } else {
+                bodyString += '<a href="javascript: void(0);" class="lang-ref-link" data-target="' + href + '">' + title + '</a>';
+              }
+            }
+          }
+        }
+      });
+      result += ',\n' + prefix + '  body: \'' + bodyString.replace(/([^\\])\\([^\\])/g, '$1\\\\$2').replace(/'/g, '\\\'').replace(/\n/g, '\' + \n' + prefix + '    \'') + '\''
+    }
+    if (topic.children.length) {
+      result += ',\n' + prefix + '  children: [\n';
+      let stringifiedChildren = [];
+      topic.children.forEach(child => {
+        stringifiedChildren.push(stringifyTopic(child, prefix + '  '))
+      });
+      result += stringifiedChildren.join(',\n');
+      result += prefix + ']';
+    } else {
+      result += ',\n' + prefix + '  children: []\n';
+    }
+    result += prefix + '}';
+    return result;
+  };
+
+  fs.readFile('../Impala/docs/impala_sqlref.ditamap', 'utf8', (err, mapRaw) => {
+    let topics = [];
+    let promises = [];
+
+    libxml.parseXmlString(mapRaw).get('//map').childNodes().forEach(x => {
+      if (x.name() === 'topicref' && x.attr('href').value() !== 'topics/impala_functions.xml') {
+        topics.push(new Topic(x.attr('href').value(), x, promises));
+      }
+    });
+
+    Promise.all(promises).then(() => {
+      fs.readFile('tools/jison/license.txt', 'utf-8', (err, licenseHeader) => {
+        let result = licenseHeader + '\n\n\n// NOTE: This is a generated file!\n// Run \'node tools/sql-docs/impalaExtractor.js\' to generate.\n\n\nvar impalaLangRefTopics = [\n';
+        let stringifiedTopics = [];
+        topics.forEach(topic => {
+          stringifiedTopics.push(stringifyTopic(topic, ''));
+        });
+        result += stringifiedTopics.join(',\n');
+        result += '\n];';
+        fs.writeFile('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js', result, () => {
+          console.log('desktop/core/src/desktop/static/desktop/js/sqlImpalaLangRef.js updated!');
+        })
+      })
+    });
+  });
+});

Some files were not shown because too many files changed in this diff