فهرست منبع

HUE-1409 [pig] Autocomplete table names

Refactored Hive autocomplete to external js file
Added HCatLoader autocomplete wizard to Pig that supports default db and any other db through dot
Enrico Berti 12 سال پیش
والد
کامیت
58993a9

+ 10 - 127
apps/beeswax/src/beeswax/templates/execute.mako

@@ -453,8 +453,11 @@ ${layout.menubar(section='query')}
 </style>
 
 <script src="/static/ext/js/jquery/plugins/jquery-fieldselection.js" type="text/javascript"></script>
+<script src="/beeswax/static/js/autocomplete.utils.js" type="text/javascript" charset="utf-8"></script>
 
 <script type="text/javascript" charset="utf-8">
+    var HIVE_AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
+
     $(document).ready(function(){
       var queryPlaceholder = "${_('Example: SELECT * FROM tablename, or press CTRL + space')}";
 
@@ -526,7 +529,7 @@ ${layout.menubar(section='query')}
 
       $("#id_query-database").change(function () {
         $.cookie("hueBeeswaxLastDatabase", $(this).val(), {expires: 90});
-        getTables(function(){}); //preload tables for current DB
+        hac_getTables($("#id_query-database").val(), function(){}); //preload tables for the default db
       });
 
       ## If no particular query is loaded
@@ -612,129 +615,9 @@ ${layout.menubar(section='query')}
         }
       }
 
-      var AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
-
-      function autocomplete(options) {
-        if (options.database == null) {
-          $.getJSON(AUTOCOMPLETE_BASE_URL, options.onDataReceived);
-        }
-        if (options.database != null) {
-          if (options.table != null) {
-            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/" + options.table, options.onDataReceived);
-          }
-          else {
-            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/", options.onDataReceived);
-          }
-        }
-      }
-
-      function getTableAliases() {
-        var _aliases = {};
-        var _val = codeMirror.getValue();
-        var _from = _val.toUpperCase().indexOf("FROM");
-        if (_from > -1) {
-          var _match = _val.toUpperCase().substring(_from).match(/ON|WHERE|GROUP|SORT/);
-          var _to = _val.length;
-          if (_match) {
-            _to = _match.index;
-          }
-          var _found = _val.substr(_from, _to).replace(/(\r\n|\n|\r)/gm, "").replace(/from/gi, "").replace(/join/gi, ",").split(",");
-          for (var i = 0; i < _found.length; i++) {
-            var _tablealias = $.trim(_found[i]).split(" ");
-            if (_tablealias.length > 1) {
-              _aliases[_tablealias[1]] = _tablealias[0];
-            }
-          }
-        }
-        return _aliases;
-      }
-
-      function getTableColumns(tableName, callback) {
-        if (tableName.indexOf("(") > -1) {
-          tableName = tableName.substr(tableName.indexOf("(") + 1);
-        }
-
-        var _aliases = getTableAliases();
-        if (_aliases[tableName]) {
-          tableName = _aliases[tableName];
-        }
-
-        if ($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName) != null) {
-          callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
-          autocomplete({
-            database: $("#id_query-database").val(),
-            table: tableName,
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
-              }
-            }
-          });
-        }
-        else {
-          autocomplete({
-            database: $("#id_query-database").val(),
-            table: tableName,
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
-                callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
-              }
-            }
-          });
-        }
-      }
-
-      function tableHasAlias(tableName) {
-        var _aliases = getTableAliases();
-        for (var alias in _aliases) {
-          if (_aliases[alias] == tableName) {
-            return true;
-          }
-        }
-        return false;
-      }
-
-      function getTables(callback) {
-        if ($.totalStorage('tables_' + $("#id_query-database").val()) != null) {
-          callback($.totalStorage('tables_' + $("#id_query-database").val()));
-          autocomplete({
-            database: $("#id_query-database").val(),
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
-              }
-            }
-          });
-        }
-        else {
-          autocomplete({
-            database: $("#id_query-database").val(),
-            onDataReceived: function (data) {
-              if (data.error) {
-                $.jHueNotify.error(data.error);
-              }
-              else {
-                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
-                callback($.totalStorage('tables_' + $("#id_query-database").val()));
-              }
-            }
-          });
-        }
-      }
-
       var queryEditor = $("#queryField")[0];
 
-      getTables(function(){}); //preload tables for current DB
+      hac_getTables($("#id_query-database").val(), function(){}); //preload tables for the default db
 
       % if app_name == 'impala':
         var AUTOCOMPLETE_SET = CodeMirror.impalaSQLHint;
@@ -756,10 +639,10 @@ ${layout.menubar(section='query')}
         });
         if ($.totalStorage('tables_' + $("#id_query-database").val()) == null) {
           CodeMirror.showHint(cm, AUTOCOMPLETE_SET);
-          getTables(function () {}); // if preload didn't work, tries again
+          hac_getTables($("#id_query-database").val(), function () {}); // if preload didn't work, tries again
         }
         else {
-          getTables(function (tables) {
+          hac_getTables($("#id_query-database").val(), function (tables) {
             CodeMirror.catalogTables = tables;
             var _before = codeMirror.getRange({line: 0, ch: 0}, {line: codeMirror.getCursor().line, ch: codeMirror.getCursor().ch}).replace(/(\r\n|\n|\r)/gm, " ");
             CodeMirror.possibleTable = false;
@@ -801,12 +684,12 @@ ${layout.menubar(section='query')}
             }
           }
           if (_foundTable != "") {
-            if (tableHasAlias(_foundTable)) {
+            if (hac_tableHasAlias(_foundTable, codeMirror.getValue())) {
               CodeMirror.possibleSoloField = false;
               CodeMirror.showHint(cm, AUTOCOMPLETE_SET);
             }
             else {
-              getTableColumns(_foundTable,
+              hac_getTableColumns($("#id_query-database").val(), _foundTable, codeMirror.getValue(),
                   function (columns) {
                     CodeMirror.catalogFields = columns;
                     CodeMirror.showHint(cm, AUTOCOMPLETE_SET);
@@ -843,7 +726,7 @@ ${layout.menubar(section='query')}
               var _partial = _line.substring(0, codeMirror.getCursor().ch);
               var _table = _partial.substring(_partial.lastIndexOf(" ") + 1, _partial.length - 1);
               if (codeMirror.getValue().toUpperCase().indexOf("FROM") > -1) {
-                getTableColumns(_table, function (columns) {
+                hac_getTableColumns($("#id_query-database").val(), _table, codeMirror.getValue(), function (columns) {
                   var _cols = columns.split(" ");
                   for (var col in _cols){
                     _cols[col] = "." + _cols[col];

+ 142 - 0
apps/beeswax/static/js/autocomplete.utils.js

@@ -0,0 +1,142 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+function hac_jsoncalls(options) {
+  if (typeof HIVE_AUTOCOMPLETE_BASE_URL != "undefined") {
+    if (options.database == null) {
+      $.getJSON(HIVE_AUTOCOMPLETE_BASE_URL, options.onDataReceived);
+    }
+    if (options.database != null) {
+      if (options.table != null) {
+        $.getJSON(HIVE_AUTOCOMPLETE_BASE_URL + options.database + "/" + options.table, options.onDataReceived);
+      }
+      else {
+        $.getJSON(HIVE_AUTOCOMPLETE_BASE_URL + options.database + "/", options.onDataReceived);
+      }
+    }
+  }
+  else {
+    try {
+      console.error("You need to specify a HIVE_AUTOCOMPLETE_BASE_URL to use the autocomplete")
+    }
+    catch (e) {
+    }
+  }
+}
+
+function hac_getTableAliases(textScanned) {
+  var _aliases = {};
+  var _val = textScanned; //codeMirror.getValue();
+  var _from = _val.toUpperCase().indexOf("FROM");
+  if (_from > -1) {
+    var _match = _val.toUpperCase().substring(_from).match(/ON|WHERE|GROUP|SORT/);
+    var _to = _val.length;
+    if (_match) {
+      _to = _match.index;
+    }
+    var _found = _val.substr(_from, _to).replace(/(\r\n|\n|\r)/gm, "").replace(/from/gi, "").replace(/join/gi, ",").split(",");
+    for (var i = 0; i < _found.length; i++) {
+      var _tablealias = $.trim(_found[i]).split(" ");
+      if (_tablealias.length > 1) {
+        _aliases[_tablealias[1]] = _tablealias[0];
+      }
+    }
+  }
+  return _aliases;
+}
+
+function hac_getTableColumns(databaseName, tableName, textScanned, callback) {
+  if (tableName.indexOf("(") > -1) {
+    tableName = tableName.substr(tableName.indexOf("(") + 1);
+  }
+
+  var _aliases = hac_getTableAliases(textScanned);
+  if (_aliases[tableName]) {
+    tableName = _aliases[tableName];
+  }
+
+  if ($.totalStorage('columns_' + databaseName + '_' + tableName) != null) {
+    callback($.totalStorage('columns_' + databaseName + '_' + tableName));
+    hac_jsoncalls({
+      database: databaseName,
+      table: tableName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('columns_' + databaseName + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
+        }
+      }
+    });
+  }
+  else {
+    hac_jsoncalls({
+      database: databaseName,
+      table: tableName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('columns_' + databaseName + '_' + tableName, (data.columns ? data.columns.join(" ") : ""));
+          callback($.totalStorage('columns_' + databaseName + '_' + tableName));
+        }
+      }
+    });
+  }
+}
+
+function hac_tableHasAlias(tableName, textScanned) {
+  var _aliases = hac_getTableAliases(textScanned);
+  for (var alias in _aliases) {
+    if (_aliases[alias] == tableName) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function hac_getTables(databaseName, callback) {
+  if ($.totalStorage('tables_' + databaseName) != null) {
+    callback($.totalStorage('tables_' + databaseName));
+    hac_jsoncalls({
+      database: databaseName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('tables_' + databaseName, data.tables.join(" "));
+        }
+      }
+    });
+  }
+  else {
+    hac_jsoncalls({
+      database: databaseName,
+      onDataReceived: function (data) {
+        if (data.error) {
+          $.jHueNotify.error(data.error);
+        }
+        else {
+          $.totalStorage('tables_' + databaseName, data.tables.join(" "));
+          callback($.totalStorage('tables_' + databaseName));
+        }
+      }
+    });
+  }
+}

+ 76 - 18
apps/pig/src/pig/templates/app.mako

@@ -576,12 +576,14 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
 <script src="/pig/static/js/utils.js" type="text/javascript" charset="utf-8"></script>
 <script src="/pig/static/js/pig.ko.js" type="text/javascript" charset="utf-8"></script>
 <script src="/static/ext/js/routie-0.3.0.min.js" type="text/javascript" charset="utf-8"></script>
-<link rel="stylesheet" href="/pig/static/css/pig.css">
 <script src="/static/ext/js/codemirror-3.11.js"></script>
-<link rel="stylesheet" href="/static/ext/css/codemirror.css">
 <script src="/static/ext/js/codemirror-pig.js"></script>
 <script src="/static/js/Source/jHue/codemirror-show-hint.js"></script>
 <script src="/static/js/Source/jHue/codemirror-pig-hint.js"></script>
+<script src="/beeswax/static/js/autocomplete.utils.js" type="text/javascript" charset="utf-8"></script>
+
+<link rel="stylesheet" href="/pig/static/css/pig.css">
+<link rel="stylesheet" href="/static/ext/css/codemirror.css">
 <link rel="stylesheet" href="/static/ext/css/codemirror-show-hint.css">
 
 <style>
@@ -616,6 +618,8 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
   var viewModel = new PigViewModel(appProperties);
   ko.applyBindings(viewModel);
 
+  var HIVE_AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
+
   $(document).ready(function () {
     viewModel.updateScripts();
 
@@ -651,6 +655,17 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       });
     }
 
+    var KLASS = "org.apache.hcatalog.pig.HCatLoader";
+
+    CodeMirror.onAutocomplete = function (data, from, to) {
+      if (CodeMirror.isHCatHint && data.indexOf(KLASS) > -1) {
+        codeMirror.replaceRange(" ", to, to);
+        codeMirror.setCursor(to);
+        CodeMirror.isHCatHint = false;
+        showHiveAutocomplete("default");
+      }
+    };
+
     CodeMirror.commands.autocomplete = function (cm) {
       $(document.body).on("contextmenu", function (e) {
         e.preventDefault(); // prevents native menu on FF for Mac from being shown
@@ -658,23 +673,43 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       storeVariables();
       var _line = codeMirror.getLine(codeMirror.getCursor().line);
       var _partial = _line.substring(0, codeMirror.getCursor().ch);
-      if (_partial.indexOf("'") > -1 && _partial.indexOf("'") == _partial.lastIndexOf("'") && (_partial.toLowerCase().indexOf("load") > -1
-          || _partial.toLowerCase().indexOf("into") > -1)) {
-        var _path = _partial.substring(_partial.lastIndexOf("'") + 1);
-        var _autocompleteUrl = "/filebrowser/view";
-        if (_path.indexOf("/") == 0) {
-          _autocompleteUrl += _path.substr(0, _path.lastIndexOf("/"));
-        }
-        else if (_path.indexOf("/") > 0) {
-          _autocompleteUrl += USER_HOME + _path.substr(0, _path.lastIndexOf("/"));
-        }
-        else {
-          _autocompleteUrl += USER_HOME;
+      if (_partial.indexOf("'") > -1 && _partial.indexOf("'") == _partial.lastIndexOf("'")) {
+        CodeMirror.isHCatHint = false;
+        CodeMirror.isTable = false;
+        if (_partial.toLowerCase().indexOf("load") > -1 || _partial.toLowerCase().indexOf("into") > -1) {
+          var _path = _partial.substring(_partial.lastIndexOf("'") + 1);
+          var _autocompleteUrl = "/filebrowser/view";
+          if (_path.indexOf("/") == 0) {
+            _autocompleteUrl += _path.substr(0, _path.lastIndexOf("/"));
+          }
+          else if (_path.indexOf("/") > 0) {
+            _autocompleteUrl += USER_HOME + _path.substr(0, _path.lastIndexOf("/"));
+          }
+          else {
+            _autocompleteUrl += USER_HOME;
+          }
+          var _showHCatHint = false;
+          if (_line.indexOf(KLASS) == -1) {
+            if (_partial.indexOf("'") == _partial.length - 1) {
+              _showHCatHint = true;
+            }
+            showHdfsAutocomplete(_autocompleteUrl + "?format=json", _showHCatHint);
+          }
+          else {
+            var _db = _partial.substring(_partial.lastIndexOf("'") + 1);
+            if (_db.indexOf(".") > -1) {
+              showHiveAutocomplete(_db.substring(0, _db.length - 1));
+            }
+            else {
+              showHiveAutocomplete("default");
+            }
+          }
         }
-        showHdfsAutocomplete(_autocompleteUrl + "?format=json");
       }
       else {
         CodeMirror.isPath = false;
+        CodeMirror.isTable = false;
+        CodeMirror.isHCatHint = false;
         CodeMirror.showHint(cm, CodeMirror.pigHint);
       }
     }
@@ -700,24 +735,34 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       },
       onKeyEvent: function (e, s) {
         if (s.type == "keyup") {
+          if (s.keyCode == 190) {
+            if (codeMirror.getValue().indexOf(KLASS) > -1) {
+              var _line = codeMirror.getLine(codeMirror.getCursor().line);
+              var _partial = _line.substring(0, codeMirror.getCursor().ch);
+              var _db = _partial.substring(_partial.lastIndexOf("'") + 1);
+              if (_partial.replace(/ /g, '').toUpperCase().indexOf("LOAD") == _partial.replace(/ /g, '').lastIndexOf("'") - 4) {
+                showHiveAutocomplete(_db.substring(0, _db.length - 1));
+              }
+            }
+          }
           if (s.keyCode == 191) {
             var _line = codeMirror.getLine(codeMirror.getCursor().line);
             var _partial = _line.substring(0, codeMirror.getCursor().ch);
             var _path = _partial.substring(_partial.lastIndexOf("'") + 1);
             if (_path[0] == "/") {
               if (_path.lastIndexOf("/") != 0) {
-                showHdfsAutocomplete("/filebrowser/view" + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json");
+                showHdfsAutocomplete("/filebrowser/view" + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json", false);
               }
             }
             else {
-              showHdfsAutocomplete("/filebrowser/view" + USER_HOME + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json");
+              showHdfsAutocomplete("/filebrowser/view" + USER_HOME + _partial.substring(_partial.lastIndexOf("'") + 1) + "?format=json", false);
             }
           }
         }
       }
     });
 
-    function showHdfsAutocomplete(path) {
+    function showHdfsAutocomplete(path, showHCatHint) {
       $.getJSON(path, function (data) {
         CodeMirror.currentFiles = [];
         if (data.error != null) {
@@ -734,6 +779,7 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
             }
           });
           CodeMirror.isPath = true;
+          CodeMirror.isHCatHint = showHCatHint;
           window.setTimeout(function () {
             CodeMirror.showHint(codeMirror, CodeMirror.pigHint);
           }, 100);  // timeout for IE8
@@ -741,6 +787,18 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       });
     }
 
+    hac_getTables("default", function(){}); //preload tables for the default db
+
+    function showHiveAutocomplete(databaseName) {
+      CodeMirror.isPath = false;
+      CodeMirror.isTable = true;
+      CodeMirror.isHCatHint = false;
+      hac_getTables(databaseName, function (tables) {
+        CodeMirror.catalogTables = tables;
+        CodeMirror.showHint(codeMirror, CodeMirror.pigHint);
+      });
+    }
+
     codeMirror.on("focus", function () {
       if (codeMirror.getValue() == LABELS.NEW_SCRIPT_CONTENT) {
         codeMirror.setValue("");

+ 3 - 1
apps/pig/src/pig/views.py

@@ -43,7 +43,9 @@ LOG = logging.getLogger(__name__)
 
 
 def app(request):
-  return render('app.mako', request, {})
+  return render('app.mako', request, {
+    'autocomplete_base_url': reverse('beeswax:autocomplete', kwargs={}),
+  })
 
 
 def scripts(request):

+ 13 - 2
desktop/core/static/js/Source/jHue/codemirror-pig-hint.js

@@ -38,7 +38,7 @@
     // If it's not a 'word-style' token, ignore the token.
 
     if (token.string.indexOf("'") == 0){
-      CodeMirror.isPath = true;
+      CodeMirror.isPath = !CodeMirror.isTable;
       token.string = token.string.substring(1, token.string.length);
     }
     if (token.string.indexOf("/") > -1){
@@ -65,7 +65,10 @@
   }
 
   CodeMirror.isPath = false;
+  CodeMirror.isTable = false;
+  CodeMirror.isHCatHint = false;
   CodeMirror.currentFiles = [];
+  CodeMirror.catalogTables = "";
 
   CodeMirror.pigHint = function (editor) {
     return scriptHint(editor, pigCaseInsensitive, function (e, cur) {
@@ -85,6 +88,8 @@
   var pigCaseSensitive = "AVG BinStorage CONCAT copyFromLocal copyToLocal COUNT DIFF MAX MIN  PigDump PigStorage SIZE SUM TextLoader TOKENIZE".split(" ");
 
   function getCompletions(token, context) {
+    var catalogTablesL = CodeMirror.catalogTables.toLowerCase().split(" ");
+
     var found = [], start = token.string, extraFound = [];
 
     function maybeAdd(str) {
@@ -108,15 +113,21 @@
     }
 
     function gatherCompletions(obj) {
-      if (CodeMirror.isPath || obj.indexOf("'") == 0){
+      if (CodeMirror.isPath || (obj.indexOf("'") == 0 && !CodeMirror.isTable)){
         forEach(CodeMirror.currentFiles, maybeAdd);
       }
+      else if (CodeMirror.isTable) {
+        forEach(catalogTablesL, maybeAddToExtra);
+      }
       else {
         forEach(pigCaseInsensitiveU, maybeAdd);
         forEach(pigCaseInsensitiveL, maybeAdd);
         forEach(pigCaseSensitive, maybeAdd);
         forEach(CodeMirror.availableVariables, maybeAddToExtra);
       }
+      if (CodeMirror.isHCatHint){
+        maybeAdd("<i class='icon-magic'></i> USING org.apache.hcatalog.pig.HCatLoader();");
+      }
     }
 
     if (context) {