Procházet zdrojové kódy

HUE-1717 [dbquery] Add table and column autocompletion

Add autocomplete.utils.js from beeswax and customize for dbquery.
Add sql hints javascript dependency.
execute.mako generates tables and columns that are passsed to the autocomplete function.
Added tests to test new APIs: tables and columns.
Abraham Elmahrek před 12 roky
rodič
revize
95d7fec

+ 30 - 0
apps/rdbms/src/rdbms/api.py

@@ -70,6 +70,36 @@ def databases(request, server):
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
+def tables(request, server, database):
+  query_server = dbms.get_query_server_config(server)
+
+  if not query_server:
+    raise Http404
+
+  db = dbms.get(request.user, query_server)
+
+  response = {
+    'tables': db.get_tables(database)
+  }
+
+  return HttpResponse(json.dumps(response), mimetype="application/json")
+
+
+def columns(request, server, database, table):
+  query_server = dbms.get_query_server_config(server)
+
+  if not query_server:
+    raise Http404
+
+  db = dbms.get(request.user, query_server)
+
+  response = {
+    'columns': db.get_columns(database, table)
+  }
+
+  return HttpResponse(json.dumps(response), mimetype="application/json")
+
+
 def execute_query(request, design_id=None):
   response = {'status': -1, 'message': ''}
 

+ 8 - 2
apps/rdbms/src/rdbms/server/dbms.py

@@ -80,11 +80,17 @@ class Rdbms(object):
     self.client = client
     self.server_type = server_type
 
+  def get_databases(self):
+    return self.client.get_databases()
+
+  def get_tables(self, database):
+    return self.client.get_tables(database)
+
   def get_table(self, database, table_name):
     return self.client.get_table(database, table_name)
 
-  def get_databases(self):
-    return self.client.get_databases()
+  def get_columns(self, database, table_name):
+    return self.client.get_columns(database, table_name)
 
   def execute_query(self, query, design):
     sql_query = query.sql_query

+ 1 - 1
apps/rdbms/src/rdbms/server/mysql_lib.py

@@ -97,7 +97,7 @@ class MySQLClient(BaseRDMSClient):
     return [row[0] for row in cursor.fetchall()]
 
 
-  def get_tables(self, database, table_names):
+  def get_tables(self, database, table_names=[]):
     cursor = self.connection.cursor()
     cursor.execute("SHOW TABLES")
     self.connection.commit()

+ 1 - 1
apps/rdbms/src/rdbms/server/oracle_lib.py

@@ -78,7 +78,7 @@ class OracleClient(BaseRDMSClient):
     return [self.query_server['name']]
 
 
-  def get_tables(self, database, table_names):
+  def get_tables(self, database, table_names=[]):
     cursor = self.connection.cursor()
     cursor.execute("SELECT table_name FROM all_tables")
     self.connection.commit()

+ 1 - 1
apps/rdbms/src/rdbms/server/postgresql_lib.py

@@ -78,7 +78,7 @@ class PostgreSQLClient(BaseRDMSClient):
     return [self._conn_params['database']]
 
 
-  def get_tables(self, database, table_names):
+  def get_tables(self, database, table_names=[]):
     # Doesn't use database and only retrieves tables for database currently in use.
     cursor = self.connection.cursor()
     cursor.execute("SELECT table_schema,table_name FROM information_schema.tables")

+ 1 - 1
apps/rdbms/src/rdbms/server/sqlite_lib.py

@@ -77,7 +77,7 @@ class SQLiteClient(BaseRDMSClient):
     return [self._conn_params['database']]
 
 
-  def get_tables(self, database, table_names):
+  def get_tables(self, database, table_names=[]):
     # Doesn't use database and only retrieves tables for database currently in use.
     cursor = self.connection.cursor()
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")

+ 74 - 21
apps/rdbms/src/rdbms/templates/execute.mako

@@ -288,11 +288,15 @@ ${ commonheader(_('Query'), app_name, user) | n,unicode }
 <script src="/static/ext/js/bootstrap-editable.min.js"></script>
 
 <script src="/static/ext/js/jquery/plugins/jquery-fieldselection.js" type="text/javascript"></script>
-<script src="/beeswax/static/js/autocomplete.utils.js" type="text/javascript" charset="utf-8"></script>
+<script src="/rdbms/static/js/autocomplete.utils.js" type="text/javascript" charset="utf-8"></script>
 
 <script type="text/javascript" charset="utf-8">
   var codeMirror, viewModel;
 
+  var RDBMS_AUTOCOMPLETE_BASE_URL = '/rdbms/api';
+  var RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON = [500, 404]; // error codes from rdbms/views.py - autocomplete
+  var RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK = $.noop;
+
   $(document).ready(function(){
 
     var queryPlaceholder = "${_('Example: SELECT * FROM tablename, or press CTRL + space')}";
@@ -321,13 +325,73 @@ ${ commonheader(_('Query'), app_name, user) | n,unicode }
 
     var AUTOCOMPLETE_SET = CodeMirror.sqlHint;
 
-    CodeMirror.onAutocomplete = function (data, from, to) {
-      if (CodeMirror.tableFieldMagic) {
-        codeMirror.replaceRange(" ", from, from);
-        codeMirror.setCursor(from);
+    CodeMirror.commands.autocomplete = function (cm) {
+      $(document.body).on("contextmenu", function (e) {
+        e.preventDefault(); // prevents native menu on FF for Mac from being shown
+      });
+
+      var options = {
+        'tables': {}
+      };
+
+      var pos = cm.cursorCoords();
+      $("<i class='fa fa-spinner fa-spin CodeMirror-spinner'></i>").css("top", pos.top + "px").css("left", (pos.left - 4) + "px").appendTo($("body"));
+
+      if ($.totalStorage('rdbms_tables_' + viewModel.server().name() + "_" + viewModel.database()) == null) {
         CodeMirror.showHint(codeMirror, AUTOCOMPLETE_SET);
+        rdbms_getTables(viewModel.server().name(), viewModel.database(), function () {}); // if preload didn't work, tries again
       }
-    };
+      else {
+        rdbms_getTables(viewModel.server().name(), viewModel.database(), function (tables) {
+          $.each(tables.split(' '), function(index, table) {
+            if (!(table in options['tables'])) {
+              options['tables'][table] = [];
+            }
+          });
+          var _before = codeMirror.getRange({line: 0, ch: 0}, {line: codeMirror.getCursor().line, ch: codeMirror.getCursor().ch}).replace(/(\r\n|\n|\r)/gm, " ");
+          if (_before.toUpperCase().indexOf("SELECT ") > -1 && _before.toUpperCase().indexOf(" FROM ") == -1 && !CodeMirror.fromDot) {
+            if (codeMirror.getValue().toUpperCase().indexOf("FROM ") > -1) {
+              fieldsAutocomplete(codeMirror, options);
+            } else {
+              CodeMirror.showHint(codeMirror, AUTOCOMPLETE_SET);
+            }
+          }
+          else {
+            if (_before.toUpperCase().indexOf("WHERE ") > -1 && !CodeMirror.fromDot && _before.match(/ON|GROUP|SORT/) == null) {
+              fieldsAutocomplete(codeMirror);
+            }
+            else {
+              CodeMirror.showHint(codeMirror, AUTOCOMPLETE_SET, options);
+            }
+          }
+        });
+      }
+    }
+
+    function fieldsAutocomplete(cm, options) {
+      try {
+        var _possibleTables = $.trim(codeMirror.getValue(" ").substr(codeMirror.getValue().toUpperCase().indexOf("FROM ") + 4)).split(" ");
+        var _foundTable = "";
+        for (var i = 0; i < _possibleTables.length; i++) {
+          if ($.trim(_possibleTables[i]) != "" && _foundTable == "") {
+            _foundTable = _possibleTables[i];
+          }
+        }
+        if (_foundTable != "" && _foundTable in options['tables']) {
+          if (rdbms_tableHasAlias(viewModel.server().name(), _foundTable, codeMirror.getValue())) {
+            CodeMirror.showHint(cm, AUTOCOMPLETE_SET, options);
+          } else {
+            rdbms_getTableColumns(viewModel.server().name(), viewModel.database(), _foundTable, codeMirror.getValue(),
+                function (columns) {
+                  options['tables'][_foundTable] = columns.split(' ');
+                  CodeMirror.showHint(cm, AUTOCOMPLETE_SET, options);
+                });
+          }
+        }
+      }
+      catch (e) {
+      }
+    }
 
     CodeMirror.fromDot = false;
 
@@ -341,7 +405,7 @@ ${ commonheader(_('Query'), app_name, user) | n,unicode }
       extraKeys: {
         "Ctrl-Space": function () {
           CodeMirror.fromDot = false;
-          CodeMirror.showHint(codeMirror, AUTOCOMPLETE_SET);
+          codeMirror.execCommand("autocomplete");
         },
         Tab: function (cm) {
           $("#executeQuery").focus();
@@ -350,21 +414,10 @@ ${ commonheader(_('Query'), app_name, user) | n,unicode }
       onKeyEvent: function (e, s) {
         if (s.type == "keyup") {
           if (s.keyCode == 190) {
-            var _line = codeMirror.getLine(codeMirror.getCursor().line);
-            var _partial = _line.substring(0, codeMirror.getCursor().ch);
-            var _table = _partial.substring(_partial.lastIndexOf(" ") + 1, _partial.length - 1);
             if (codeMirror.getValue().toUpperCase().indexOf("FROM") > -1) {
-              hac_getTableColumns($("#id_query-database").val(), _table, codeMirror.getValue(), function (columns) {
-                var _cols = columns.split(" ");
-                for (var col in _cols){
-                  _cols[col] = "." + _cols[col];
-                }
-                CodeMirror.catalogFields = _cols.join(" ");
-                CodeMirror.fromDot = true;
-                window.setTimeout(function () {
-                  codeMirror.execCommand("autocomplete");
-                }, 100);  // timeout for IE8
-              });
+              window.setTimeout(function () {
+                codeMirror.execCommand("autocomplete");
+              }, 100);  // timeout for IE8
             }
           }
         }

+ 14 - 0
apps/rdbms/src/rdbms/tests.py

@@ -107,6 +107,20 @@ class TestAPI(TestSQLiteRdbmsBase):
     response_dict = json.loads(response.content)
     assert_true(self.database in response_dict['databases'], response_dict)
 
+  def test_get_tables(self):
+    response = self.client.get(reverse('rdbms:api_tables', args=['sqlitee', self.database]))
+    response_dict = json.loads(response.content)
+    assert_true('test1' in response_dict['tables'], response_dict)
+
+  def test_get_columns(self):
+    response = self.client.get(reverse('rdbms:api_columns', args=['sqlitee', self.database, 'test1']))
+    response_dict = json.loads(response.content)
+    assert_true('date' in response_dict['columns'], response_dict)
+    assert_true('trans' in response_dict['columns'], response_dict)
+    assert_true('symbol' in response_dict['columns'], response_dict)
+    assert_true('qty' in response_dict['columns'], response_dict)
+    assert_true('price' in response_dict['columns'], response_dict)
+
   def test_execute_query(self):
     data = {
       'server': 'sqlitee',

+ 2 - 0
apps/rdbms/src/rdbms/urls.py

@@ -28,6 +28,8 @@ urlpatterns = patterns('rdbms.views',
 urlpatterns += patterns('rdbms.api',
   url(r'^api/servers/?$', 'servers', name='api_servers'),
   url(r'^api/servers/(?P<server>\w+)/databases/?$', 'databases', name='api_databases'),
+  url(r'^api/servers/(?P<server>\w+)/databases/(?P<database>.+?)/tables/?$', 'tables', name='api_tables'),
+  url(r'^api/servers/(?P<server>\w+)/databases/(?P<database>.+?)/tables/(?P<table>\w+)/columns/?$', 'columns', name='api_columns'),
   url(r'^api/query/((?P<design_id>\d+)/?)?$', 'save_query', name='api_save_query'),
   url(r'^api/query/(?P<design_id>\d+)/get$', 'fetch_saved_query', name='api_fetch_saved_query'),
   url(r'^api/execute/(?P<design_id>\d+)?$', 'execute_query', name='api_execute_query'),

+ 181 - 0
apps/rdbms/static/js/autocomplete.utils.js

@@ -0,0 +1,181 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+function rdbms_jsoncalls(options) {
+  if (typeof RDBMS_AUTOCOMPLETE_BASE_URL != "undefined") {
+    if (options.database === null) {
+      $.getJSON(RDBMS_AUTOCOMPLETE_BASE_URL, options.onDataReceived);
+    }
+    if (options.database) {
+      if (options.table) {
+        $.getJSON(RDBMS_AUTOCOMPLETE_BASE_URL + "/servers/" + options.server + "/databases/" + options.database + "/tables/" + options.table + "/columns", options.onDataReceived);
+      }
+      else {
+        $.getJSON(RDBMS_AUTOCOMPLETE_BASE_URL + "/servers/" + options.server + "/databases/" + options.database + "/tables", options.onDataReceived);
+      }
+    }
+  }
+  else {
+    try {
+      console.error("You need to specify a RDBMS_AUTOCOMPLETE_BASE_URL to use the autocomplete");
+    }
+    catch (e) {
+    }
+  }
+}
+
+function rdbms_hasExpired(timestamp){
+  var TIME_TO_LIVE_IN_MILLIS = 600000; // 10 minutes
+  return (new Date()).getTime() - timestamp > TIME_TO_LIVE_IN_MILLIS;
+}
+
+function rdbms_getTableAliases(textScanned) {
+  var _aliases = {};
+  var _val = textScanned; //codeMirror.getValue();
+  var _from = _val.toUpperCase().indexOf("FROM ");
+  if (_from > -1) {
+    var _match = _val.toUpperCase().substring(_from).match(/ON|WHERE|GROUP|SORT|ORDER/);
+    var _to = _val.length;
+    if (_match) {
+      _to = _match.index;
+    }
+    var _found = _val.substr(_from, _to).replace(/(\r\n|\n|\r)/gm, "").replace(/from/gi, "").replace(/join/gi, ",").split(",");
+    for (var i = 0; i < _found.length; i++) {
+      var _tablealias = $.trim(_found[i]).split(" ");
+      if (_tablealias.length > 1) {
+        _aliases[_tablealias[1]] = _tablealias[0];
+      }
+    }
+  }
+  return _aliases;
+}
+
+function rdbms_tableHasAlias(tableName, textScanned) {
+  var _aliases = rdbms_getTableAliases(textScanned);
+  for (var alias in _aliases) {
+    if (_aliases[alias] == tableName) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function rdbms_getTableColumns(serverName, databaseName, tableName, textScanned, callback) {
+  if (tableName.indexOf("(") > -1) {
+    tableName = tableName.substr(tableName.indexOf("(") + 1);
+  }
+
+  var _aliases = rdbms_getTableAliases(textScanned);
+  if (_aliases[tableName]) {
+    tableName = _aliases[tableName];
+  }
+
+  if ($.totalStorage('rdbms_columns_' + serverName + '_' + databaseName + '_' + tableName) && $.totalStorage('rdbms_extended_columns_' + serverName + '_' + databaseName + '_' + tableName)) {
+    callback($.totalStorage('rdbms_columns_' + serverName + '_' + databaseName + '_' + tableName), $.totalStorage('rdbms_extended_columns_' + serverName + '_' + databaseName + '_' + tableName));
+    if ($.totalStorage('rdbms_timestamp_columns_' + serverName + '_' + databaseName + '_' + tableName) === null || rdbms_hasExpired($.totalStorage('rdbms_timestamp_columns_' + serverName + '_' + databaseName + '_' + tableName))){
+      rdbms_jsoncalls({
+        server: serverName,
+        database: databaseName,
+        table: tableName,
+        onDataReceived: function (data) {
+          if (typeof RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK == "function") {
+            RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK(data);
+          }
+          if (data.error) {
+            if (typeof RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON == "undefined" || data.code === null || RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON.indexOf(data.code) == -1){
+              $(document).trigger('error', data.error);
+            }
+          }
+          else {
+            $.totalStorage('rdbms_columns_' + serverName + '_' + databaseName + '_' + tableName, (data.columns ? "* " + data.columns.join(" ") : "*"));
+            $.totalStorage('rdbms_extended_columns_' + serverName + '_' + databaseName + '_' + tableName, (data.extended_columns ? data.extended_columns : []));
+            $.totalStorage('rdbms_timestamp_columns_' + serverName + '_' + databaseName + '_' + tableName, (new Date()).getTime());
+          }
+        }
+      });
+    }
+  }
+  else {
+    rdbms_jsoncalls({
+      server: serverName,
+      database: databaseName,
+      table: tableName,
+      onDataReceived: function (data) {
+        if (typeof RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK == "function") {
+          RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK(data);
+        }
+        if (data.error) {
+          if (typeof RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON == "undefined" || data.code === null || RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON.indexOf(data.code) == -1){
+            $(document).trigger('error', data.error);
+          }
+        }
+        else {
+          $.totalStorage('rdbms_columns_' + serverName + '_' + databaseName + '_' + tableName, (data.columns ? "* " + data.columns.join(" ") : "*"));
+          $.totalStorage('rdbms_extended_columns_' + serverName + '_' + databaseName + '_' + tableName, (data.extended_columns ? data.extended_columns : []));
+          $.totalStorage('rdbms_timestamp_columns_' + serverName + '_' + databaseName + '_' + tableName, (new Date()).getTime());
+          callback($.totalStorage('rdbms_columns_' + serverName + '_' + databaseName + '_' + tableName), $.totalStorage('rdbms_extended_columns_' + serverName + '_' + databaseName + '_' + tableName));
+        }
+      }
+    });
+  }
+}
+
+function rdbms_getTables(serverName, databaseName, callback) {
+  if ($.totalStorage('rdbms_tables_' + serverName + '_' + databaseName)) {
+    callback($.totalStorage('rdbms_tables_' + serverName + '_' + databaseName));
+    if ($.totalStorage('rdbms_timestamp_tables_' + serverName + '_' + databaseName) === null || rdbms_hasExpired($.totalStorage('rdbms_timestamp_tables_' + serverName + '_' + databaseName))){
+      rdbms_jsoncalls({
+        server: serverName,
+        database: databaseName,
+        onDataReceived: function (data) {
+          if (typeof RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK == "function") {
+            RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK(data);
+          }
+          if (data.error) {
+            if (typeof RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON == "undefined" || data.code === null || RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON.indexOf(data.code) == -1){
+              $(document).trigger('error', data.error);
+            }
+          }
+          else {
+            $.totalStorage('rdbms_tables_' + serverName + '_' + databaseName, data.tables.join(" "));
+            $.totalStorage('rdbms_timestamp_tables_' + serverName + '_' + databaseName, (new Date()).getTime());
+          }
+        }
+      });
+    }
+  }
+  else {
+    rdbms_jsoncalls({
+      server: serverName,
+      database: databaseName,
+      onDataReceived: function (data) {
+        if (typeof RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK == "function") {
+          RDBMS_AUTOCOMPLETE_GLOBAL_CALLBACK(data);
+        }
+        if (data.error) {
+          if (typeof RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON == "undefined" || data.code === null || RDBMS_AUTOCOMPLETE_FAILS_SILENTLY_ON.indexOf(data.code) == -1){
+            $(document).trigger('error', data.error);
+          }
+        }
+        else {
+          $.totalStorage('rdbms_tables_' + serverName + '_' + databaseName, data.tables.join(" "));
+          $.totalStorage('rdbms_timestamp_tables_' + serverName + '_' + databaseName, (new Date()).getTime());
+          callback($.totalStorage('rdbms_tables_' + serverName + '_' + databaseName));
+        }
+      }
+    });
+  }
+}