Explorar o código

HUE-1409 [pig] Autocomplete table names

Refactored Hive autocomplete to external js file
Added HCatLoader autocomplete wizard to Pig that supports default db and any other db through dot
Enrico Berti %!s(int64=12) %!d(string=hai) anos
pai
achega
58993a9

+ 10 - 127
apps/beeswax/src/beeswax/templates/execute.mako

@@ -453,8 +453,11 @@ ${layout.menubar(section='query')}
 </style>
 
 <script src="/static/ext/js/jquery/plugins/jquery-fieldselection.js" type="text/javascript"></script>
+<script src="/beeswax/static/js/autocomplete.utils.js" type="text/javascript" charset="utf-8"></script>
 
 <script type="text/javascript" charset="utf-8">
+    var HIVE_AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
+
     $(document).ready(function(){
       var queryPlaceholder = "${_('Example: SELECT * FROM tablename, or press CTRL + space')}";
 
@@ -526,7 +529,7 @@ ${layout.menubar(section='query')}
 
       $("#id_query-database").change(function () {
         $.cookie("hueBeeswaxLastDatabase", $(this).val(), {expires: 90});
-        getTables(function(){}); //preload tables for current DB
+        hac_getTables($("#id_query-database").val(), function(){}); //preload tables for the default db
       });
 
       ## If no particular query is loaded
@@ -612,129 +615,9 @@ ${layout.menubar(section='query')}
         }
       }
 
-      var AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
-
-      function autocomplete(options) {
-        if (options.database == null) {
-          $.getJSON(AUTOCOMPLETE_BASE_URL, options.onDataReceived);
-        }
-        if (options.database != null) {
-          if (options.table != null) {
-            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/" + options.table, options.onDataReceived);
-          }
-          else {
-            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/", options.onDataReceived);
-          }
-        }
-      }
-
-      function getTableAliases() {
-        var _aliases = {};
-        var _val = codeMirror.getValue();
-        var _from = _val.toUpperCase().indexOf("FROM");
-        if (_from > -1) {
-          var _match = _val.toUpperCase().substring(_from).match(/ON|WHERE|GROUP|SORT/);
-          var _to = _val.length;
-          if (_match) {
-            _to = _match.index;
-          }
-          var _found = _val.substr(_from, _to).replace(/(\r\n|\n|\r)/gm, "").replace(/from/gi, "").replace(/join/gi, ",").split(",");
-          for (var i = 0; i < _found.length; i++) {
-            var _tablealias = $.trim(_found[i]).split(" ");
-            if (_tablealias.length > 1) {
-              _aliases[_tablealias[1]] = _tablealias[0];
-            }
-          }
-        }
-        return _aliases;
-      }
-
-      function getTableColumns(tableName, callback) {
-        if (tableName.indexOf("(") > -1) {
-          tableName = tableName.substr(tableName.indexOf("(") + 1);
-        }
-
-        var _aliases = getTableAliases();
-        if (_aliases[tableName]) {
-          tableName = _aliases[tableName];
-        }
-
-        if ($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName) != null) {
-          callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
-          autocomplete({
-            database: $("#id_query-database").val(),
-            table: tableName,
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
-              }
-            }
-          });
-        }
-        else {
-          autocomplete({
-            database: $("#id_query-database").val(),
-            table: tableName,
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
-                callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
-              }
-            }
-          });
-        }
-      }
-
-      function tableHasAlias(tableName) {
-        var _aliases = getTableAliases();
-        for (var alias in _aliases) {
-          if (_aliases[alias] == tableName) {
-            return true;
-          }
-        }
-        return false;
-      }
-
-      function getTables(callback) {
-        if ($.totalStorage('tables_' + $("#id_query-database").val()) != null) {
-          callback($.totalStorage('tables_' + $("#id_query-database").val()));
-          autocomplete({
-            database: $("#id_query-database").val(),
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
-              }
-            }
-          });
-        }
-        else {
-          autocomplete({
-            database: $("#id_query-database").val(),
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
-                callback($.totalStorage('tables_' + $("#id_query-database").val()));
-              }
-            }
-          });
-        }
-      }
-
       var queryEditor = $("#queryField")[0];
 
-      getTables(function(){}); //preload tables for current DB
+      hac_getTables($("#id_query-database").val(), function(){}); //preload tables for the default db
 
       % if app_name == 'impala':
         var AUTOCOMPLETE_SET = CodeMirror.impalaSQLHint;
@@ -756,10 +639,10 @@ ${layout.menubar(section='query')}
         });
         if ($.totalStorage('tables_' + $("#id_query-database").val()) == null) {
           CodeMirror.showHint(cm, AUTOCOMPLETE_SET);
-          getTables(function () {}); // if preload didn't work, tries again
+          hac_getTables($("#id_query-database").val(), function () {}); // if preload didn't work, tries again
         }
         else {
-          getTables(function (tables) {
+          hac_getTables($("#id_query-database").val(), function (tables) {
             CodeMirror.catalogTables = tables;
             var _before = codeMirror.getRange({line: 0, ch: 0}, {line: codeMirror.getCursor().line, ch: codeMirror.getCursor().ch}).replace(/(\r\n|\n|\r)/gm, " ");
             CodeMirror.possibleTable = false;
@@ -801,12 +684,12 @@ ${layout.menubar(section='query')}
             }
           }
           if (_foundTable != "") {
-            if (tableHasAlias(_foundTable)) {
+            if (hac_tableHasAlias(_foundTable, codeMirror.getValue())) {
               CodeMirror.possibleSoloField = false;
               CodeMirror.showHint(cm, AUTOCOMPLETE_SET);
             }
             else {
-              getTableColumns(_foundTable,
+              hac_getTableColumns($("#id_query-database").val(), _foundTable, codeMirror.getValue(),
                   function (columns) {
                     CodeMirror.catalogFields = columns;
                     CodeMirror.showHint(cm, AUTOCOMPLETE_SET);
@@ -843,7 +726,7 @@ ${layout.menubar(section='query')}
               var _partial = _line.substring(0, codeMirror.getCursor().ch);
               var _table = _partial.substring(_partial.lastIndexOf(" ") + 1, _partial.length - 1);
               if (codeMirror.getValue().toUpperCase().indexOf("FROM") > -1) {
-                getTableColumns(_table, function (columns) {
+                hac_getTableColumns($("#id_query-database").val(), _table, codeMirror.getValue(), function (columns) {
                   var _cols = columns.split(" ");
                   for (var col in _cols){
                     _cols[col] = "." + _cols[col];

+ 142 - 0
apps/beeswax/static/js/autocomplete.utils.js

@@ -0,0 +1,142 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+function hac_jsoncalls(options) {
+  if (typeof HIVE_AUTOCOMPLETE_BASE_URL != "undefined") {
+    if (options.database == null) {
+      $.getJSON(HIVE_AUTOCOMPLETE_BASE_URL, options.onDataReceived);
+    }
+    if (options.database != null) {
+      if (options.table != null) {
+        $.getJSON(HIVE_AUTOCOMPLETE_BASE_URL + options.database + "/" + options.table, options.onDataReceived);
+      }
+      else {
+        $.getJSON(HIVE_AUTOCOMPLETE_BASE_URL + options.database + "/", options.onDataReceived);
+      }
+    }
+  }
+  else {
+    try {
+      console.error("You need to specify a HIVE_AUTOCOMPLETE_BASE_URL to use the autocomplete")
+    }
+    catch (e) {
+    }
+  }
+}
+
+function hac_getTableAliases(textScanned) {
+  var _aliases = {};
+  var _val = textScanned; //codeMirror.getValue();
+  var _from = _val.toUpperCase().indexOf("FROM");
+  if (_from > -1) {
+    var _match = _val.toUpperCase().substring(_from).match(/ON|WHERE|GROUP|SORT/);
+    var _to = _val.length;
+    if (_match) {
+      _to = _match.index;
+    }
+    var _found = _val.substr(_from, _to).replace(/(\r\n|\n|\r)/gm, "").replace(/from/gi, "").replace(/join/gi, ",").split(",");
+    for (var i = 0; i < _found.length; i++) {
+      var _tablealias = $.trim(_found[i]).split(" ");
+      if (_tablealias.length > 1) {
+        _aliases[_tablealias[1]] = _tablealias[0];
+      }
+    }
+  }
+  return _aliases;
+}
+
+function hac_getTableColumns(databaseName, tableName, textScanned, callback) {
+  if (tableName.indexOf("(") > -1) {
+    tableName = tableName.substr(tableName.indexOf("(") + 1);
+  }
+
+  var _aliases = hac_getTableAliases(textScanned);
+  if (_aliases[tableName]) {
+    tableName = _aliases[tableName];
+  }
+
+  if ($.totalStorage('columns_' + databaseName + '_' + tableName) != null) {
+    callback($.totalStorage('columns_' + databaseName + '_' + tableName));
+    hac_jsoncalls({
+      database: databaseName,
+      table: tableName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('columns_' + databaseName + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
+        }
+      }
+    });
+  }
+  else {
+    hac_jsoncalls({
+      database: databaseName,
+      table: tableName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('columns_' + databaseName + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
+          callback($.totalStorage('columns_' + databaseName + '_' + tableName));
+        }
+      }
+    });
+  }
+}
+
+function hac_tableHasAlias(tableName, textScanned) {
+  var _aliases = hac_getTableAliases(textScanned);
+  for (var alias in _aliases) {
+    if (_aliases[alias] == tableName) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function hac_getTables(databaseName, callback) {
+  if ($.totalStorage('tables_' + databaseName) != null) {
+    callback($.totalStorage('tables_' + databaseName));
+    hac_jsoncalls({
+      database: databaseName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('tables_' + databaseName, data.tables.join(" "));
+        }
+      }
+    });
+  }
+  else {
+    hac_jsoncalls({
+      database: databaseName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('tables_' + databaseName, data.tables.join(" "));
+          callback($.totalStorage('tables_' + databaseName));
+        }
+      }
+    });
+  }
+}

+ 76 - 18
apps/pig/src/pig/templates/app.mako

@@ -576,12 +576,14 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
 <script src="/pig/static/js/utils.js" type="text/javascript" charset="utf-8"></script>
 <script src="/pig/static/js/pig.ko.js" type="text/javascript" charset="utf-8"></script>
 <script src="/static/ext/js/routie-0.3.0.min.js" type="text/javascript" charset="utf-8"></script>
-<link rel="stylesheet" href="/pig/static/css/pig.css">
 <script src="/static/ext/js/codemirror-3.11.js"></script>
-<link rel="stylesheet" href="/static/ext/css/codemirror.css">
 <script src="/static/ext/js/codemirror-pig.js"></script>
 <script src="/static/js/Source/jHue/codemirror-show-hint.js"></script>
 <script src="/static/js/Source/jHue/codemirror-pig-hint.js"></script>
+<script src="/beeswax/static/js/autocomplete.utils.js" type="text/javascript" charset="utf-8"></script>
+
+<link rel="stylesheet" href="/pig/static/css/pig.css">
+<link rel="stylesheet" href="/static/ext/css/codemirror.css">
 <link rel="stylesheet" href="/static/ext/css/codemirror-show-hint.css">
 
 <style>
@@ -616,6 +618,8 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
   var viewModel = new PigViewModel(appProperties);
   ko.applyBindings(viewModel);
 
+  var HIVE_AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
+
   $(document).ready(function () {
     viewModel.updateScripts();
 
@@ -651,6 +655,17 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       });
     }
 
+    var KLASS = "org.apache.hcatalog.pig.HCatLoader";
+
+    CodeMirror.onAutocomplete = function (data, from, to) {
+      if (CodeMirror.isHCatHint && data.indexOf(KLASS) > -1) {
+        codeMirror.replaceRange(" ", to, to);
+        codeMirror.setCursor(to);
+        CodeMirror.isHCatHint = false;
+        showHiveAutocomplete("default");
+      }
+    };
+
     CodeMirror.commands.autocomplete = function (cm) {
       $(document.body).on("contextmenu", function (e) {
         e.preventDefault(); // prevents native menu on FF for Mac from being shown
@@ -658,23 +673,43 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       storeVariables();
       var _line = codeMirror.getLine(codeMirror.getCursor().line);
       var _partial = _line.substring(0, codeMirror.getCursor().ch);
-      if (_partial.indexOf("'") > -1 && _partial.indexOf("'") == _partial.lastIndexOf("'") && (_partial.toLowerCase().indexOf("load") > -1
-          || _partial.toLowerCase().indexOf("into") > -1)) {
-        var _path = _partial.substring(_partial.lastIndexOf("'") + 1);
-        var _autocompleteUrl = "/filebrowser/view";
-        if (_path.indexOf("/") == 0) {
-          _autocompleteUrl += _path.substr(0, _path.lastIndexOf("/"));
-        }
-        else if (_path.indexOf("/") > 0) {
-          _autocompleteUrl += USER_HOME + _path.substr(0, _path.lastIndexOf("/"));
-        }
-        else {
-          _autocompleteUrl += USER_HOME;
+      if (_partial.indexOf("'") > -1 && _partial.indexOf("'") == _partial.lastIndexOf("'")) {
+        CodeMirror.isHCatHint = false;
+        CodeMirror.isTable = false;
+        if (_partial.toLowerCase().indexOf("load") > -1 || _partial.toLowerCase().indexOf("into") > -1) {
+          var _path = _partial.substring(_partial.lastIndexOf("'") + 1);
+          var _autocompleteUrl = "/filebrowser/view";
+          if (_path.indexOf("/") == 0) {
+            _autocompleteUrl += _path.substr(0, _path.lastIndexOf("/"));
+          }
+          else if (_path.indexOf("/") > 0) {
+            _autocompleteUrl += USER_HOME + _path.substr(0, _path.lastIndexOf("/"));
+          }
+          else {
+            _autocompleteUrl += USER_HOME;
+          }
+          var _showHCatHint = false;
+          if (_line.indexOf(KLASS) == -1) {
+            if (_partial.indexOf("'") == _partial.length - 1) {
+              _showHCatHint = true;
+            }
+            showHdfsAutocomplete(_autocompleteUrl + "?format=json", _showHCatHint);
+          }
+          else {
+            var _db = _partial.substring(_partial.lastIndexOf("'") + 1);
+            if (_db.indexOf(".") > -1) {
+              showHiveAutocomplete(_db.substring(0, _db.length - 1));
+            }
+            else {
+              showHiveAutocomplete("default");
+            }
+          }
         }
-        showHdfsAutocomplete(_autocompleteUrl + "?format=json");
       }
       else {
         CodeMirror.isPath = false;
+        CodeMirror.isTable = false;
+        CodeMirror.isHCatHint = false;
         CodeMirror.showHint(cm, CodeMirror.pigHint);
       }
     }
@@ -700,24 +735,34 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       },
       onKeyEvent: function (e, s) {
         if (s.type == "keyup") {
+          if (s.keyCode == 190) {
+            if (codeMirror.getValue().indexOf(KLASS) > -1) {
+              var _line = codeMirror.getLine(codeMirror.getCursor().line);
+              var _partial = _line.substring(0, codeMirror.getCursor().ch);
+              var _db = _partial.substring(_partial.lastIndexOf("'") + 1);
+              if (_partial.replace(/ /g, '').toUpperCase().indexOf("LOAD") == _partial.replace(/ /g, '').lastIndexOf("'") - 4) {
+                showHiveAutocomplete(_db.substring(0, _db.length - 1));
+              }
+            }
+          }
           if (s.keyCode == 191) {
             var _line = codeMirror.getLine(codeMirror.getCursor().line);
             var _partial = _line.substring(0, codeMirror.getCursor().ch);
             var _path = _partial.substring(_partial.lastIndexOf("'") + 1);
             if (_path[0] == "/") {
               if (_path.lastIndexOf("/") != 0) {
-                showHdfsAutocomplete("/filebrowser/view" + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json");
+                showHdfsAutocomplete("/filebrowser/view" + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json", false);
               }
             }
             else {
-              showHdfsAutocomplete("/filebrowser/view" + USER_HOME + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json");
+              showHdfsAutocomplete("/filebrowser/view" + USER_HOME + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json", false);
             }
           }
         }
       }
     });
 
-    function showHdfsAutocomplete(path) {
+    function showHdfsAutocomplete(path, showHCatHint) {
       $.getJSON(path, function (data) {
         CodeMirror.currentFiles = [];
         if (data.error != null) {
@@ -734,6 +779,7 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
             }
           });
           CodeMirror.isPath = true;
+          CodeMirror.isHCatHint = showHCatHint;
           window.setTimeout(function () {
             CodeMirror.showHint(codeMirror, CodeMirror.pigHint);
           }, 100);  // timeout for IE8
@@ -741,6 +787,18 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       });
     }
 
+    hac_getTables("default", function(){}); //preload tables for the default db
+
+    function showHiveAutocomplete(databaseName) {
+      CodeMirror.isPath = false;
+      CodeMirror.isTable = true;
+      CodeMirror.isHCatHint = false;
+      hac_getTables(databaseName, function (tables) {
+        CodeMirror.catalogTables = tables;
+        CodeMirror.showHint(codeMirror, CodeMirror.pigHint);
+      });
+    }
+
     codeMirror.on("focus", function () {
       if (codeMirror.getValue() == LABELS.NEW_SCRIPT_CONTENT) {
         codeMirror.setValue("");

+ 3 - 1
apps/pig/src/pig/views.py

@@ -43,7 +43,9 @@ LOG = logging.getLogger(__name__)
 
 
 def app(request):
-  return render('app.mako', request, {})
+  return render('app.mako', request, {
+    'autocomplete_base_url': reverse('beeswax:autocomplete', kwargs={}),
+  })
 
 
 def scripts(request):

+ 13 - 2
desktop/core/static/js/Source/jHue/codemirror-pig-hint.js

@@ -38,7 +38,7 @@
     // If it's not a 'word-style' token, ignore the token.
 
     if (token.string.indexOf("'") == 0){
-      CodeMirror.isPath = true;
+      CodeMirror.isPath = !CodeMirror.isTable;
       token.string = token.string.substring(1, token.string.length);
     }
     if (token.string.indexOf("/") > -1){
@@ -65,7 +65,10 @@
   }
 
   CodeMirror.isPath = false;
+  CodeMirror.isTable = false;
+  CodeMirror.isHCatHint = false;
   CodeMirror.currentFiles = [];
+  CodeMirror.catalogTables = "";
 
   CodeMirror.pigHint = function (editor) {
     return scriptHint(editor, pigCaseInsensitive, function (e, cur) {
@@ -85,6 +88,8 @@
   var pigCaseSensitive = "AVG BinStorage CONCAT copyFromLocal copyToLocal COUNT DIFF MAX MIN  PigDump PigStorage SIZE SUM TextLoader TOKENIZE".split(" ");
 
   function getCompletions(token, context) {
+    var catalogTablesL = CodeMirror.catalogTables.toLowerCase().split(" ");
+
     var found = [], start = token.string, extraFound = [];
 
     function maybeAdd(str) {
@@ -108,15 +113,21 @@
     }
 
     function gatherCompletions(obj) {
-      if (CodeMirror.isPath || obj.indexOf("'") == 0){
+      if (CodeMirror.isPath || (obj.indexOf("'") == 0 && !CodeMirror.isTable)){
         forEach(CodeMirror.currentFiles, maybeAdd);
       }
+      else if (CodeMirror.isTable) {
+        forEach(catalogTablesL, maybeAddToExtra);
+      }
       else {
         forEach(pigCaseInsensitiveU, maybeAdd);
         forEach(pigCaseInsensitiveL, maybeAdd);
         forEach(pigCaseSensitive, maybeAdd);
         forEach(CodeMirror.availableVariables, maybeAddToExtra);
       }
+      if (CodeMirror.isHCatHint){
+        maybeAdd("<i class='icon-magic'></i> USING org.apache.hcatalog.pig.HCatLoader();");
+      }
     }
 
     if (context) {