Browse Source

HUE-1189 [beeswax] Lazily build autocomplete tree

The autocomplete is requested and built on demand
Support for local storage to speed up autocomplete window
Enrico Berti 12 năm trước cách đây
mục cha
commit
710e8e209f

+ 129 - 62
apps/beeswax/src/beeswax/templates/execute.mako

@@ -236,7 +236,7 @@ ${layout.menubar(section='query')}
                               <i class="icon-question-sign" id="help"></i>
                               <div id="help-content" class="hide">
                                 <ul style="text-align: left;">
-                                  <li>${ _('Press Shift + Space to autocomplete') }</li>
+                                  <li>${ _('Press CTRL + Space to autocomplete') }</li>
                                   <li>${ _("You can execute queries with multiple SQL statements delimited by a semicolon ';'") }</li>
                                   <li>${ _('You can highlight and run a fragment of a query') }</li>
                                 </ul>
@@ -431,11 +431,12 @@ ${layout.menubar(section='query')}
 </style>
 
 <script src="/static/ext/js/jquery/plugins/jquery.cookie.js"></script>
+<script src="/static/ext/js/jquery/plugins/jquery.total-storage.min.js"></script>
 <script src="/static/ext/js/jquery/plugins/jquery-fieldselection.js" type="text/javascript"></script>
 
 <script type="text/javascript" charset="utf-8">
     $(document).ready(function(){
-      var queryPlaceholder = "${_('Example: SELECT * FROM tablename')}";
+      var queryPlaceholder = "${_('Example: SELECT * FROM tablename, or press CTRL + space')}";
 
       var successfunction = function(response, newValue) {
         if (response.status != 0) {
@@ -585,7 +586,21 @@ ${layout.menubar(section='query')}
         }
       }
 
-      var dbTree = ${ autocomplete | n,unicode };
+      var AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
+
+      function autocomplete(options) {
+        if (options.database == null) {
+          $.getJSON(AUTOCOMPLETE_BASE_URL, options.onDataReceived);
+        }
+        if (options.database != null) {
+          if (options.table != null) {
+            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/" + options.table, options.onDataReceived);
+          }
+          else {
+            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/", options.onDataReceived);
+          }
+        }
+      }
 
       function getTableAliases() {
         var _aliases = {};
@@ -608,29 +623,46 @@ ${layout.menubar(section='query')}
         return _aliases;
       }
 
-      function getTableFields(tableName, withDot) {
+      function getTableColumns(tableName, callback) {
         if (tableName.indexOf("(") > -1) {
           tableName = tableName.substr(tableName.indexOf("(") + 1);
         }
-        var _branch = dbTree[$("#id_query-database").val()];
-        if (_branch != null && _branch.tables) {
-          for (var i = 0; i < _branch.tables.length; i++) {
-            var _aliases = getTableAliases();
-            if (_aliases[tableName]) {
-              tableName = _aliases[tableName];
+
+        var _aliases = getTableAliases();
+        if (_aliases[tableName]) {
+          tableName = _aliases[tableName];
+        }
+
+        if ($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName) != null) {
+          callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
+          autocomplete({
+            database: $("#id_query-database").val(),
+            table: tableName,
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, data.columns.join(" "));
+              }
             }
-            if (_branch.tables[i].name == tableName) {
-              var _cols = _branch.tables[i].cols.slice(0);
-              if (withDot) {
-                for (var idx = 0; idx < _cols.length; idx++) {
-                  _cols[idx] = "." + _cols[idx];
-                }
+          });
+        }
+        else {
+          autocomplete({
+            database: $("#id_query-database").val(),
+            table: tableName,
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, data.columns.join(" "));
+                callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
               }
-              return _cols.join(" ");
             }
-          }
+          });
         }
-        return "";
       }
 
       function tableHasAlias(tableName) {
@@ -643,54 +675,79 @@ ${layout.menubar(section='query')}
         return false;
       }
 
-      function getTables() {
-        var _branch = dbTree[$("#id_query-database").val()];
-        if (_branch != null && _branch.tables) {
-          var _names = "";
-          for (var i = 0; i < _branch.tables.length; i++) {
-            if (i > 0) {
-              _names += " ";
+      function getTables(callback) {
+        if ($.totalStorage('tables_' + $("#id_query-database").val()) != null) {
+          callback($.totalStorage('tables_' + $("#id_query-database").val()));
+          autocomplete({
+            database: $("#id_query-database").val(),
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
+              }
             }
-            _names += _branch.tables[i].name;
-          }
-          return _names;
+          });
+        }
+        else {
+          autocomplete({
+            database: $("#id_query-database").val(),
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
+                callback($.totalStorage('tables_' + $("#id_query-database").val()));
+              }
+            }
+          });
         }
-        return "";
       }
 
       var queryEditor = $("#queryField")[0];
 
       CodeMirror.commands.autocomplete = function (cm) {
-        CodeMirror.catalogTables = getTables();
-        var _before = codeMirror.getRange({line: 0, ch: 0}, {line: codeMirror.getCursor().line, ch: codeMirror.getCursor().ch}).replace(/(\r\n|\n|\r)/gm, " ");
-        CodeMirror.possibleTable = false;
-        if (_before.toUpperCase().indexOf(" FROM ") > -1 && _before.toUpperCase().indexOf(" ON ") == -1 && _before.toUpperCase().indexOf(" WHERE ") == -1) {
-          CodeMirror.possibleTable = true;
-        }
-        CodeMirror.possibleSoloField = false;
-        if (_before.toUpperCase().indexOf("SELECT ") > -1 && _before.toUpperCase().indexOf(" FROM ") && !CodeMirror.fromDot) {
-          CodeMirror.possibleSoloField = true;
-          try {
-            var _possibleTables = $.trim(codeMirror.getValue().substr(codeMirror.getValue().toUpperCase().indexOf("FROM") + 4)).split(" ");
-            var _foundTable = "";
-            for (var i = 0; i < _possibleTables.length; i++) {
-              if ($.trim(_possibleTables[i]) != "" && _foundTable == "") {
-                _foundTable = _possibleTables[i];
-              }
-            }
-            if (_foundTable != "") {
-              if (tableHasAlias(_foundTable)) {
-                CodeMirror.possibleSoloField = false;
+        getTables(function (tables) {
+          CodeMirror.catalogTables = tables;
+          var _before = codeMirror.getRange({line: 0, ch: 0}, {line: codeMirror.getCursor().line, ch: codeMirror.getCursor().ch}).replace(/(\r\n|\n|\r)/gm, " ");
+          CodeMirror.possibleTable = false;
+          if (_before.toUpperCase().indexOf(" FROM ") > -1 && _before.toUpperCase().indexOf(" ON ") == -1 && _before.toUpperCase().indexOf(" WHERE ") == -1) {
+            CodeMirror.possibleTable = true;
+          }
+          CodeMirror.possibleSoloField = false;
+          if (_before.toUpperCase().indexOf("SELECT ") > -1 && _before.toUpperCase().indexOf(" FROM ") == -1 && !CodeMirror.fromDot) {
+            CodeMirror.possibleSoloField = true;
+            try {
+              var _possibleTables = $.trim(codeMirror.getValue().substr(codeMirror.getValue().toUpperCase().indexOf("FROM") + 4)).split(" ");
+              var _foundTable = "";
+              for (var i = 0; i < _possibleTables.length; i++) {
+                if ($.trim(_possibleTables[i]) != "" && _foundTable == "") {
+                  _foundTable = _possibleTables[i];
+                }
               }
-              else {
-                CodeMirror.catalogFields = getTableFields(_foundTable, false);
+              if (_foundTable != "") {
+                if (tableHasAlias(_foundTable)) {
+                  CodeMirror.possibleSoloField = false;
+                  CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
+                }
+                else {
+                  getTableColumns(_foundTable,
+                          function (columns) {
+                            CodeMirror.catalogFields = columns;
+                            CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
+                          });
+                }
               }
             }
+            catch (e) {
+            }
           }
-          catch (e) {
+          else {
+            CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
           }
-        }
-        CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
+        });
       }
 
       CodeMirror.fromDot = false;
@@ -703,6 +760,10 @@ ${layout.menubar(section='query')}
         lineNumbers: true,
         mode: "text/x-hiveql",
         extraKeys: {
+          "Shift-Space": function () {
+            CodeMirror.fromDot = false;
+            codeMirror.execCommand("autocomplete");
+          },
           "Ctrl-Space": function () {
             CodeMirror.fromDot = false;
             codeMirror.execCommand("autocomplete");
@@ -714,14 +775,20 @@ ${layout.menubar(section='query')}
         onKeyEvent: function (e, s) {
           if (s.type == "keyup") {
             if (s.keyCode == 190) {
-              window.setTimeout(function () {
-                var _line = codeMirror.getLine(codeMirror.getCursor().line);
-                var _partial = _line.substring(0, codeMirror.getCursor().ch);
-                var _table = _partial.substring(_partial.lastIndexOf(" ") + 1, _partial.length - 1);
-                CodeMirror.catalogFields = getTableFields(_table, true);
+              var _line = codeMirror.getLine(codeMirror.getCursor().line);
+              var _partial = _line.substring(0, codeMirror.getCursor().ch);
+              var _table = _partial.substring(_partial.lastIndexOf(" ") + 1, _partial.length - 1);
+              getTableColumns(_table, function (columns) {
+                var _cols = columns.split(" ");
+                for (var col in _cols){
+                  _cols[idx] = "." + _cols[idx];
+                }
+                CodeMirror.catalogFields = _cols.join(" ");
                 CodeMirror.fromDot = true;
-                codeMirror.execCommand("autocomplete");
-              }, 100); // timeout for IE8
+                window.setTimeout(function () {
+                  codeMirror.execCommand("autocomplete");
+                }, 100);  // timeout for IE8
+              });
             }
           }
         }

+ 5 - 0
apps/beeswax/src/beeswax/urls.py

@@ -30,6 +30,11 @@ urlpatterns = patterns('beeswax.views',
   url(r'^save_results/(?P<id>\d+)$', 'save_results', name='save_results'),
   url(r'^save_design_properties$', 'save_design_properties', name='save_design_properties'), # Ajax
 
+  url(r'^autocomplete/$', 'autocomplete', name='autocomplete'),
+  url(r'^autocomplete/(?P<database>\w+)/$', 'autocomplete', name='autocomplete'),
+  url(r'^autocomplete/(?P<database>\w+)/(?P<table>\w+)$', 'autocomplete', name='autocomplete'),
+
+
   url(r'^my_queries$', 'my_queries', name='my_queries'),
   url(r'^list_designs$', 'list_designs', name='list_designs'),
   url(r'^list_trashed_designs$', 'list_trashed_designs', name='list_trashed_designs'),

+ 24 - 23
apps/beeswax/src/beeswax/views.py

@@ -381,25 +381,6 @@ def execute_query(request, design_id=None):
   dbs = db.get_databases()
   databases = ((db, db) for db in dbs)
 
-  # Todo: build lazily
-  autocomplete = {}
-  for database in dbs:
-    tables = db.get_tables(database=database)
-    autocomplete_tables = []
-    for table in tables:
-      try:
-        t = db.get_table(database, table)
-        autocomplete_tables.append({
-          'name': t.name,
-          'cols': [column.name for column in t.cols]
-        })
-      except Exception, e:
-        LOG.warn('Skipping table %s.%s : %s' % (database, table, e))
-
-    autocomplete[database] = {
-      'tables': autocomplete_tables
-    }
-
   if request.method == 'POST':
     form.bind(request.POST)
     form.query.fields['database'].choices =  databases # Could not do it in the form
@@ -454,7 +435,7 @@ def execute_query(request, design_id=None):
     'error_message': error_message,
     'form': form,
     'log': log,
-    'autocomplete': json.dumps(autocomplete),
+    'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
     'on_success_url': on_success_url,
     'can_edit_name': design and not design.is_auto and design.name,
   })
@@ -854,7 +835,7 @@ def confirm_query(request, query, on_success_url=None):
     'design': None,
     'on_success_url': on_success_url,
     'design': None,
-    'autocomplete': json.dumps({}),
+    'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
   })
 
 
@@ -948,6 +929,27 @@ def query_done_cb(request, server_id):
   return HttpResponse(message_template % message)
 
 
+def autocomplete(request, database=None, table=None):
+  app_name = get_app_name(request)
+  query_server = get_query_server_config(app_name)
+  db = dbms.get(request.user, query_server)
+  response = {}
+
+  try:
+    if database is None:
+      response['databases'] = db.get_databases()
+    elif table is None:
+      response['tables'] = db.get_tables(database=database)
+    else:
+      t = db.get_table(database, table)
+      response['columns'] = [column.name for column in t.cols]
+  except Exception, e:
+    LOG.warn('Autocomplete data fetching error %s.%s: %s' % (database, table, e))
+    response['error'] = e.message
+
+
+  return HttpResponse(json.dumps(response), mimetype="application/json")
+
 
 """
 Utils
@@ -1082,7 +1084,7 @@ def _run_parameterized_query(request, design_id, explain):
         'error_message': error_message,
         'form': query_form,
         'log': log,
-        'autocomplete': json.dumps({}),
+        'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
       })
   else:
     return render("parameterization.mako", request, dict(form=parameterization_form, design=design, explain=explain))
@@ -1399,4 +1401,3 @@ def _update_query_state(query_history):
 WHITESPACE = re.compile("\s+", re.MULTILINE)
 def collapse_whitespace(s):
   return WHITESPACE.sub(" ", s).strip()
-

+ 22 - 0
desktop/core/static/ext/js/jquery/plugins/jquery.total-storage.min.js

@@ -0,0 +1,22 @@
+/*
+ * TotalStorage
+ *
+ * Copyright (c) 2012 Jared Novack & Upstatement (upstatement.com)
+ * Dual licensed under the MIT and GPL licenses:
+ * http://www.opensource.org/licenses/mit-license.php
+ * http://www.gnu.org/licenses/gpl.html
+ *
+ * Total Storage is the conceptual the love child of jStorage by Andris Reinman,
+ * and Cookie by Klaus Hartl -- though this is not connected to either project.
+ *
+ * @name $.totalStorage
+ * @cat Plugins/Cookie
+ * @author Jared Novack/jared@upstatement.com
+ * @version 1.1.2
+ * @url http://upstatement.com/blog/2012/01/jquery-local-storage-done-right-and-easy/
+ */
+
+(function(c,h){var d="undefined"===typeof window.localStorage?h:window.localStorage,e;e="undefined"==typeof d||"undefined"==typeof window.JSON?!1:!0;c.totalStorage=function(b,a){return c.totalStorage.impl.init(b,a)};c.totalStorage.setItem=function(b,a){return c.totalStorage.impl.setItem(b,a)};c.totalStorage.getItem=function(b){return c.totalStorage.impl.getItem(b)};c.totalStorage.getAll=function(){return c.totalStorage.impl.getAll()};c.totalStorage.deleteItem=function(b){return c.totalStorage.impl.deleteItem(b)};
+c.totalStorage.impl={init:function(b,a){return"undefined"!=typeof a?this.setItem(b,a):this.getItem(b)},setItem:function(b,a){if(!e)try{return c.cookie(b,a),a}catch(g){console.log("Local Storage not supported by this browser. Install the cookie plugin on your site to take advantage of the same functionality. You can get it at https://github.com/carhartl/jquery-cookie")}var f=JSON.stringify(a);d.setItem(b,f);return this.parseResult(f)},getItem:function(b){if(!e)try{return this.parseResult(c.cookie(b))}catch(a){return null}b=
+d.getItem(b);return this.parseResult(b)},deleteItem:function(b){if(!e)try{return c.cookie(b,null),!0}catch(a){return!1}d.removeItem(b);return!0},getAll:function(){var b=[];if(e)for(var a in d)a.length&&b.push({key:a,value:this.parseResult(d.getItem(a))});else try{var g=document.cookie.split(";");for(a=0;a<g.length;a++){var f=g[a].split("=")[0];b.push({key:f,value:this.parseResult(c.cookie(f))})}}catch(h){return null}return b},parseResult:function(b){var a;try{a=JSON.parse(b),"undefined"==typeof a&&
+(a=b),"true"==a&&(a=!0),"false"==a&&(a=!1),parseFloat(a)==a&&"object"!=typeof a&&(a=parseFloat(a))}catch(c){a=b}return a}}})(jQuery);