Bläddra i källkod

HUE-1189 [beeswax] Lazily build autocomplete tree

The autocomplete is requested and built on demand
Support for local storage to speed up autocomplete window
Enrico Berti 12 år sedan
förälder
incheckning
710e8e209f

+ 129 - 62
apps/beeswax/src/beeswax/templates/execute.mako

@@ -236,7 +236,7 @@ ${layout.menubar(section='query')}
                               <i class="icon-question-sign" id="help"></i>
                               <div id="help-content" class="hide">
                                 <ul style="text-align: left;">
-                                  <li>${ _('Press Shift + Space to autocomplete') }</li>
+                                  <li>${ _('Press CTRL + Space to autocomplete') }</li>
                                   <li>${ _("You can execute queries with multiple SQL statements delimited by a semicolon ';'") }</li>
                                   <li>${ _('You can highlight and run a fragment of a query') }</li>
                                 </ul>
@@ -431,11 +431,12 @@ ${layout.menubar(section='query')}
 </style>
 
 <script src="/static/ext/js/jquery/plugins/jquery.cookie.js"></script>
+<script src="/static/ext/js/jquery/plugins/jquery.total-storage.min.js"></script>
 <script src="/static/ext/js/jquery/plugins/jquery-fieldselection.js" type="text/javascript"></script>
 
 <script type="text/javascript" charset="utf-8">
     $(document).ready(function(){
-      var queryPlaceholder = "${_('Example: SELECT * FROM tablename')}";
+      var queryPlaceholder = "${_('Example: SELECT * FROM tablename, or press CTRL + space')}";
 
       var successfunction = function(response, newValue) {
         if (response.status != 0) {
@@ -585,7 +586,21 @@ ${layout.menubar(section='query')}
         }
       }
 
-      var dbTree = ${ autocomplete | n,unicode };
+      var AUTOCOMPLETE_BASE_URL = "${ autocomplete_base_url | n,unicode }";
+
+      function autocomplete(options) {
+        if (options.database == null) {
+          $.getJSON(AUTOCOMPLETE_BASE_URL, options.onDataReceived);
+        }
+        if (options.database != null) {
+          if (options.table != null) {
+            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/" + options.table, options.onDataReceived);
+          }
+          else {
+            $.getJSON(AUTOCOMPLETE_BASE_URL + options.database + "/", options.onDataReceived);
+          }
+        }
+      }
 
       function getTableAliases() {
         var _aliases = {};
@@ -608,29 +623,46 @@ ${layout.menubar(section='query')}
         return _aliases;
       }
 
-      function getTableFields(tableName, withDot) {
+      function getTableColumns(tableName, callback) {
         if (tableName.indexOf("(") > -1) {
           tableName = tableName.substr(tableName.indexOf("(") + 1);
         }
-        var _branch = dbTree[$("#id_query-database").val()];
-        if (_branch != null && _branch.tables) {
-          for (var i = 0; i < _branch.tables.length; i++) {
-            var _aliases = getTableAliases();
-            if (_aliases[tableName]) {
-              tableName = _aliases[tableName];
+
+        var _aliases = getTableAliases();
+        if (_aliases[tableName]) {
+          tableName = _aliases[tableName];
+        }
+
+        if ($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName) != null) {
+          callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
+          autocomplete({
+            database: $("#id_query-database").val(),
+            table: tableName,
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, data.columns.join(" "));
+              }
             }
-            if (_branch.tables[i].name == tableName) {
-              var _cols = _branch.tables[i].cols.slice(0);
-              if (withDot) {
-                for (var idx = 0; idx < _cols.length; idx++) {
-                  _cols[idx] = "." + _cols[idx];
-                }
+          });
+        }
+        else {
+          autocomplete({
+            database: $("#id_query-database").val(),
+            table: tableName,
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName, data.columns.join(" "));
+                callback($.totalStorage('columns_' + $("#id_query-database").val() + '_' + tableName));
               }
-              return _cols.join(" ");
             }
-          }
+          });
         }
-        return "";
       }
 
       function tableHasAlias(tableName) {
@@ -643,54 +675,79 @@ ${layout.menubar(section='query')}
         return false;
       }
 
-      function getTables() {
-        var _branch = dbTree[$("#id_query-database").val()];
-        if (_branch != null && _branch.tables) {
-          var _names = "";
-          for (var i = 0; i < _branch.tables.length; i++) {
-            if (i > 0) {
-              _names += " ";
+      function getTables(callback) {
+        if ($.totalStorage('tables_' + $("#id_query-database").val()) != null) {
+          callback($.totalStorage('tables_' + $("#id_query-database").val()));
+          autocomplete({
+            database: $("#id_query-database").val(),
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
+              }
             }
-            _names += _branch.tables[i].name;
-          }
-          return _names;
+          });
+        }
+        else {
+          autocomplete({
+            database: $("#id_query-database").val(),
+            onDataReceived: function (data) {
+              if (data.error) {
+                $.jHueNotify.error(data.error);
+              }
+              else {
+                $.totalStorage('tables_' + $("#id_query-database").val(), data.tables.join(" "));
+                callback($.totalStorage('tables_' + $("#id_query-database").val()));
+              }
+            }
+          });
         }
-        return "";
       }
 
       var queryEditor = $("#queryField")[0];
 
       CodeMirror.commands.autocomplete = function (cm) {
-        CodeMirror.catalogTables = getTables();
-        var _before = codeMirror.getRange({line: 0, ch: 0}, {line: codeMirror.getCursor().line, ch: codeMirror.getCursor().ch}).replace(/(\r\n|\n|\r)/gm, " ");
-        CodeMirror.possibleTable = false;
-        if (_before.toUpperCase().indexOf(" FROM ") > -1 && _before.toUpperCase().indexOf(" ON ") == -1 && _before.toUpperCase().indexOf(" WHERE ") == -1) {
-          CodeMirror.possibleTable = true;
-        }
-        CodeMirror.possibleSoloField = false;
-        if (_before.toUpperCase().indexOf("SELECT ") > -1 && _before.toUpperCase().indexOf(" FROM ") && !CodeMirror.fromDot) {
-          CodeMirror.possibleSoloField = true;
-          try {
-            var _possibleTables = $.trim(codeMirror.getValue().substr(codeMirror.getValue().toUpperCase().indexOf("FROM") + 4)).split(" ");
-            var _foundTable = "";
-            for (var i = 0; i < _possibleTables.length; i++) {
-              if ($.trim(_possibleTables[i]) != "" && _foundTable == "") {
-                _foundTable = _possibleTables[i];
-              }
-            }
-            if (_foundTable != "") {
-              if (tableHasAlias(_foundTable)) {
-                CodeMirror.possibleSoloField = false;
+        getTables(function (tables) {
+          CodeMirror.catalogTables = tables;
+          var _before = codeMirror.getRange({line: 0, ch: 0}, {line: codeMirror.getCursor().line, ch: codeMirror.getCursor().ch}).replace(/(\r\n|\n|\r)/gm, " ");
+          CodeMirror.possibleTable = false;
+          if (_before.toUpperCase().indexOf(" FROM ") > -1 && _before.toUpperCase().indexOf(" ON ") == -1 && _before.toUpperCase().indexOf(" WHERE ") == -1) {
+            CodeMirror.possibleTable = true;
+          }
+          CodeMirror.possibleSoloField = false;
+          if (_before.toUpperCase().indexOf("SELECT ") > -1 && _before.toUpperCase().indexOf(" FROM ") == -1 && !CodeMirror.fromDot) {
+            CodeMirror.possibleSoloField = true;
+            try {
+              var _possibleTables = $.trim(codeMirror.getValue().substr(codeMirror.getValue().toUpperCase().indexOf("FROM") + 4)).split(" ");
+              var _foundTable = "";
+              for (var i = 0; i < _possibleTables.length; i++) {
+                if ($.trim(_possibleTables[i]) != "" && _foundTable == "") {
+                  _foundTable = _possibleTables[i];
+                }
               }
-              else {
-                CodeMirror.catalogFields = getTableFields(_foundTable, false);
+              if (_foundTable != "") {
+                if (tableHasAlias(_foundTable)) {
+                  CodeMirror.possibleSoloField = false;
+                  CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
+                }
+                else {
+                  getTableColumns(_foundTable,
+                          function (columns) {
+                            CodeMirror.catalogFields = columns;
+                            CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
+                          });
+                }
               }
             }
+            catch (e) {
+            }
           }
-          catch (e) {
+          else {
+            CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
           }
-        }
-        CodeMirror.showHint(cm, CodeMirror.hiveQLHint);
+        });
       }
 
       CodeMirror.fromDot = false;
@@ -703,6 +760,10 @@ ${layout.menubar(section='query')}
         lineNumbers: true,
         mode: "text/x-hiveql",
         extraKeys: {
+          "Shift-Space": function () {
+            CodeMirror.fromDot = false;
+            codeMirror.execCommand("autocomplete");
+          },
           "Ctrl-Space": function () {
             CodeMirror.fromDot = false;
             codeMirror.execCommand("autocomplete");
@@ -714,14 +775,20 @@ ${layout.menubar(section='query')}
         onKeyEvent: function (e, s) {
           if (s.type == "keyup") {
             if (s.keyCode == 190) {
-              window.setTimeout(function () {
-                var _line = codeMirror.getLine(codeMirror.getCursor().line);
-                var _partial = _line.substring(0, codeMirror.getCursor().ch);
-                var _table = _partial.substring(_partial.lastIndexOf(" ") + 1, _partial.length - 1);
-                CodeMirror.catalogFields = getTableFields(_table, true);
+              var _line = codeMirror.getLine(codeMirror.getCursor().line);
+              var _partial = _line.substring(0, codeMirror.getCursor().ch);
+              var _table = _partial.substring(_partial.lastIndexOf(" ") + 1, _partial.length - 1);
+              getTableColumns(_table, function (columns) {
+                var _cols = columns.split(" ");
+                for (var col in _cols){
+                  _cols[idx] = "." + _cols[idx];
+                }
+                CodeMirror.catalogFields = _cols.join(" ");
                 CodeMirror.fromDot = true;
-                codeMirror.execCommand("autocomplete");
-              }, 100); // timeout for IE8
+                window.setTimeout(function () {
+                  codeMirror.execCommand("autocomplete");
+                }, 100);  // timeout for IE8
+              });
             }
           }
         }

+ 5 - 0
apps/beeswax/src/beeswax/urls.py

@@ -30,6 +30,11 @@ urlpatterns = patterns('beeswax.views',
   url(r'^save_results/(?P<id>\d+)$', 'save_results', name='save_results'),
   url(r'^save_design_properties$', 'save_design_properties', name='save_design_properties'), # Ajax
 
+  url(r'^autocomplete/$', 'autocomplete', name='autocomplete'),
+  url(r'^autocomplete/(?P<database>\w+)/$', 'autocomplete', name='autocomplete'),
+  url(r'^autocomplete/(?P<database>\w+)/(?P<table>\w+)$', 'autocomplete', name='autocomplete'),
+
+
   url(r'^my_queries$', 'my_queries', name='my_queries'),
   url(r'^list_designs$', 'list_designs', name='list_designs'),
   url(r'^list_trashed_designs$', 'list_trashed_designs', name='list_trashed_designs'),

+ 24 - 23
apps/beeswax/src/beeswax/views.py

@@ -381,25 +381,6 @@ def execute_query(request, design_id=None):
   dbs = db.get_databases()
   databases = ((db, db) for db in dbs)
 
-  # Todo: build lazily
-  autocomplete = {}
-  for database in dbs:
-    tables = db.get_tables(database=database)
-    autocomplete_tables = []
-    for table in tables:
-      try:
-        t = db.get_table(database, table)
-        autocomplete_tables.append({
-          'name': t.name,
-          'cols': [column.name for column in t.cols]
-        })
-      except Exception, e:
-        LOG.warn('Skipping table %s.%s : %s' % (database, table, e))
-
-    autocomplete[database] = {
-      'tables': autocomplete_tables
-    }
-
   if request.method == 'POST':
     form.bind(request.POST)
     form.query.fields['database'].choices =  databases # Could not do it in the form
@@ -454,7 +435,7 @@ def execute_query(request, design_id=None):
     'error_message': error_message,
     'form': form,
     'log': log,
-    'autocomplete': json.dumps(autocomplete),
+    'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
     'on_success_url': on_success_url,
     'can_edit_name': design and not design.is_auto and design.name,
   })
@@ -854,7 +835,7 @@ def confirm_query(request, query, on_success_url=None):
     'design': None,
     'on_success_url': on_success_url,
     'design': None,
-    'autocomplete': json.dumps({}),
+    'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
   })
 
 
@@ -948,6 +929,27 @@ def query_done_cb(request, server_id):
   return HttpResponse(message_template % message)
 
 
+def autocomplete(request, database=None, table=None):
+  app_name = get_app_name(request)
+  query_server = get_query_server_config(app_name)
+  db = dbms.get(request.user, query_server)
+  response = {}
+
+  try:
+    if database is None:
+      response['databases'] = db.get_databases()
+    elif table is None:
+      response['tables'] = db.get_tables(database=database)
+    else:
+      t = db.get_table(database, table)
+      response['columns'] = [column.name for column in t.cols]
+  except Exception, e:
+    LOG.warn('Autocomplete data fetching error %s.%s: %s' % (database, table, e))
+    response['error'] = e.message
+
+
+  return HttpResponse(json.dumps(response), mimetype="application/json")
+
 
 """
 Utils
@@ -1082,7 +1084,7 @@ def _run_parameterized_query(request, design_id, explain):
         'error_message': error_message,
         'form': query_form,
         'log': log,
-        'autocomplete': json.dumps({}),
+        'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
       })
   else:
     return render("parameterization.mako", request, dict(form=parameterization_form, design=design, explain=explain))
@@ -1399,4 +1401,3 @@ def _update_query_state(query_history):
 WHITESPACE = re.compile("\s+", re.MULTILINE)
 def collapse_whitespace(s):
   return WHITESPACE.sub(" ", s).strip()
-

+ 22 - 0
desktop/core/static/ext/js/jquery/plugins/jquery.total-storage.min.js

@@ -0,0 +1,22 @@
+/*
+ * TotalStorage
+ *
+ * Copyright (c) 2012 Jared Novack & Upstatement (upstatement.com)
+ * Dual licensed under the MIT and GPL licenses:
+ * http://www.opensource.org/licenses/mit-license.php
+ * http://www.gnu.org/licenses/gpl.html
+ *
+ * Total Storage is the conceptual the love child of jStorage by Andris Reinman,
+ * and Cookie by Klaus Hartl -- though this is not connected to either project.
+ *
+ * @name $.totalStorage
+ * @cat Plugins/Cookie
+ * @author Jared Novack/jared@upstatement.com
+ * @version 1.1.2
+ * @url http://upstatement.com/blog/2012/01/jquery-local-storage-done-right-and-easy/
+ */
+
+(function(c,h){var d="undefined"===typeof window.localStorage?h:window.localStorage,e;e="undefined"==typeof d||"undefined"==typeof window.JSON?!1:!0;c.totalStorage=function(b,a){return c.totalStorage.impl.init(b,a)};c.totalStorage.setItem=function(b,a){return c.totalStorage.impl.setItem(b,a)};c.totalStorage.getItem=function(b){return c.totalStorage.impl.getItem(b)};c.totalStorage.getAll=function(){return c.totalStorage.impl.getAll()};c.totalStorage.deleteItem=function(b){return c.totalStorage.impl.deleteItem(b)};
+c.totalStorage.impl={init:function(b,a){return"undefined"!=typeof a?this.setItem(b,a):this.getItem(b)},setItem:function(b,a){if(!e)try{return c.cookie(b,a),a}catch(g){console.log("Local Storage not supported by this browser. Install the cookie plugin on your site to take advantage of the same functionality. You can get it at https://github.com/carhartl/jquery-cookie")}var f=JSON.stringify(a);d.setItem(b,f);return this.parseResult(f)},getItem:function(b){if(!e)try{return this.parseResult(c.cookie(b))}catch(a){return null}b=
+d.getItem(b);return this.parseResult(b)},deleteItem:function(b){if(!e)try{return c.cookie(b,null),!0}catch(a){return!1}d.removeItem(b);return!0},getAll:function(){var b=[];if(e)for(var a in d)a.length&&b.push({key:a,value:this.parseResult(d.getItem(a))});else try{var g=document.cookie.split(";");for(a=0;a<g.length;a++){var f=g[a].split("=")[0];b.push({key:f,value:this.parseResult(c.cookie(f))})}}catch(h){return null}return b},parseResult:function(b){var a;try{a=JSON.parse(b),"undefined"==typeof a&&
+(a=b),"true"==a&&(a=!0),"false"==a&&(a=!1),parseFloat(a)==a&&"object"!=typeof a&&(a=parseFloat(a))}catch(c){a=b}return a}}})(jQuery);