Browse Source

HUE-1810 [sqoop] autocomplete tables and fields

Abraham Elmahrek 12 years ago
parent
commit
282d218

+ 77 - 0
apps/sqoop/src/sqoop/api/autocomplete.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from django.http import HttpResponse, Http404
+
+from librdbms import conf as librdbms_conf
+from librdbms.server import dbms
+
+
+def get_query_server_config_from_request(request):
+  check_params = {
+    'server_name': request.GET.get('server_type')
+  }
+  if 'host' in request.GET:
+    check_params['server_host'] = request.GET.get('host')
+  if 'port' in request.GET:
+    check_params['server_port'] = request.GET.get('port')
+  if 'username' in request.GET:
+    check_params['username'] = request.GET.get('username')
+
+  for alias in librdbms_conf.DATABASES:
+    valid = True
+    config = dbms.get_query_server_config(alias)
+    for param in check_params:
+      if config[param] != check_params[param]:
+        valid = False
+    if valid:
+      return config
+
+  return None
+
+
+def autocomplete(request, database=None, table=None):
+  response = {
+    'status': 0,
+    'errors': []
+  }
+
+  if 'server_type' not in request.GET:
+    raise Http404()
+
+  query_server = get_query_server_config_from_request(request)
+  if database:
+    query_server['name'] = database
+
+  if not query_server:
+    raise Http404()
+
+  db = dbms.get(request.user, query_server)
+
+  if database:
+    response['databases'] = [database]
+    if table:
+      response['tables'] = [table]
+      response['columns'] = db.get_columns(database, table)
+    else:
+      response['tables'] = db.get_tables(database)
+  else:
+    response['databases'] = db.get_databases()
+
+  return HttpResponse(json.dumps(response), mimetype="application/json")

+ 45 - 1
apps/sqoop/src/sqoop/templates/app.mako

@@ -218,7 +218,7 @@ ${ commonheader(None, "sqoop", user) | n,unicode }
             <div class="controls">
               <select class="input-xlarge" name="connector" data-bind="'options': $root.connectors, 'optionsText': function(item) { return item.name(); }, 'optionsValue': function(item) { return item.id(); }, 'value': connector_id">
               </select>
-              </div>
+            </div>
           </div>
           <fieldset data-bind="foreach: connector">
             <div data-bind="foreach: inputs">
@@ -600,6 +600,7 @@ ${ commonheader(None, "sqoop", user) | n,unicode }
 <script src="/static/ext/js/knockout.x-editable.js"></script>
 <script src="/sqoop/static/js/cclass.js" type="text/javascript" charset="utf-8"></script>
 <script src="/sqoop/static/js/koify.js" type="text/javascript" charset="utf-8"></script>
+<script src="/sqoop/static/js/sqoop.autocomplete.js" type="text/javascript" charset="utf-8"></script>
 <script src="/sqoop/static/js/sqoop.utils.js" type="text/javascript" charset="utf-8"></script>
 <script src="/sqoop/static/js/sqoop.wizard.js" type="text/javascript" charset="utf-8"></script>
 <script src="/sqoop/static/js/sqoop.forms.js" type="text/javascript" charset="utf-8"></script>
@@ -771,6 +772,47 @@ $(document).on('delete_fail.job', handle_form_errors);
 $(document).on('show_section', function(e, section){
   viewModel.shownSection(section);
 });
+$(document).on('changed.page', function(e, jobWizard) {
+  $('input[name="table.tableName').typeahead({
+    'source': function(query, process) {
+      var database = viewModel.connection().database();
+      switch (viewModel.connection().jdbcDriver()) {
+        case 'com.mysql.jdbc.Driver':
+        return autocomplete.tables('mysql', database);
+        break;
+        case 'org.postgresql.Driver':
+        return autocomplete.tables('postgresql', database);
+        break;
+        case 'oracle.jdbc.OracleDriver':
+        return autocomplete.tables('oracle', database);
+      }
+      return [];
+    }
+  });
+  function columnsAutocomplete(query, process) {
+    var database = viewModel.connection().database();
+    if (viewModel.job()) {
+      var table = viewModel.job().table();
+      switch (viewModel.connection().jdbcDriver()) {
+        case 'com.mysql.jdbc.Driver':
+        return autocomplete.columns('mysql', database, table);
+        break;
+        case 'org.postgresql.Driver':
+        return autocomplete.columns('postgresql', database, table);
+        break;
+        case 'oracle.jdbc.OracleDriver':
+        return autocomplete.columns('oracle', database, table);
+      }
+      return [];
+    }
+  }
+  $('input[name="table.columns"]').typeahead({
+    'source': columnsAutocomplete
+  });
+  $('input[name="table.partitionColumn"]').typeahead({
+    'source': columnsAutocomplete
+  });
+});
 
 $(document).on('keyup', 'input#filter', function() {
   viewModel.filter($('#filter').val());
@@ -962,6 +1004,7 @@ $(document).ready(function () {
       } else {
         viewModel.jobWizard.index(viewModel.jobWizard.getIndex(page));
       }
+      $(document).trigger('changed.page', [viewModel.jobWizard]);
       $("*[rel=tooltip]").tooltip({
         placement: 'right'
       });
@@ -973,6 +1016,7 @@ $(document).ready(function () {
         $("*[rel=tooltip]").tooltip({
           placement: 'right'
         });
+        routie('job/edit/wizard/' + 0);
       });
     },
     "job/new": function () {

+ 3 - 0
apps/sqoop/src/sqoop/urls.py

@@ -22,6 +22,9 @@ urlpatterns = patterns('sqoop.views',
 )
 
 urlpatterns += patterns('sqoop.api',
+  url(r'^api/autocomplete/databases/?$', 'autocomplete', name='autocomplete_databases'),
+  url(r'^api/autocomplete/databases/(?P<database>.+)/tables/?$', 'autocomplete', name='autocomplete_tables'),
+  url(r'^api/autocomplete/databases/(?P<database>.+)/tables/(?P<table>.+)/columns/?$', 'autocomplete', name='autocomplete_fields'),
   url(r'^api/framework/?$', 'framework', name='framework'),
   url(r'^api/connectors/?$', 'connectors', name='connectors'),
   url(r'^api/connectors/resources/?$', 'connectors_resources', name='connectors_resources'),

+ 110 - 0
apps/sqoop/static/js/sqoop.autocomplete.js

@@ -0,0 +1,110 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+var autocomplete = (function($) {
+  function cas(server_type, database, table, value) {
+    var key = '';
+    if (server_type) {
+      if (database) {
+        if (table) {
+          key = 'sqoop_' + server_type + "_" + database + "_" + table;
+        } else {
+          key = 'sqoop_' + server_type + "_" + database;
+        }
+      } else {
+        key = 'sqoop_' + server_type;
+      }
+    }
+    var val = $.totalStorage(key) || [];
+    if (value) {
+      $.totalStorage(key, value);
+    }
+    return val;
+  }
+
+  var loading = {
+    'databases': false,
+    'tables': false,
+    'columns': false
+  };
+
+  return {
+    'databases': function(server_type) {
+      if (!loading.databases && server_type) {
+        loading.databases = true;
+        $(document).trigger('fetch.databases', []);
+        var request = {
+          url: '/sqoop/api/autocomplete/databases/?server_type=' + server_type,
+          dataType: 'json',
+          type: 'GET',
+          success: function(data) {
+            var databases = cas(server_type, null, null, data.databases);
+            $(document).trigger('fetched.databases', [databases]);
+          },
+          error: function() {},
+          always: function() {
+            loading.databases = false;
+          }
+        };
+        $.ajax(request);
+      }
+      return cas(server_type);
+    },
+    'tables': function(server_type, database) {
+      if (!loading.tables && server_type && database) {
+        loading.tables = true;
+        $(document).trigger('fetch.tables', []);
+        var request = {
+          url: '/sqoop/api/autocomplete/databases/' + database + '/tables?server_type=' + server_type,
+          dataType: 'json',
+          type: 'GET',
+          success: function(data) {
+            var tables = cas(server_type, database, null, data.tables);
+            $(document).trigger('fetched.tables', [tables]);
+          },
+          error: function() {},
+          always: function() {
+            loading.tables = false;
+          }
+        };
+        $.ajax(request);
+      }
+      return cas(server_type, database);
+    },
+    'columns': function(server_type, database, table) {
+      if (!loading.columns && server_type && database && table) {
+        loading.columns = true;
+        $(document).trigger('fetch.columns', []);
+        var request = {
+          url: '/sqoop/api/autocomplete/databases/' + database + '/tables/' + table + '/columns?server_type=' + server_type,
+          dataType: 'json',
+          type: 'GET',
+          success: function(data) {
+            var columns = cas(server_type, database, table, data.columns);
+            $(document).trigger('fetched.columns', [columns]);
+          },
+          error: function() {},
+          always: function() {
+            loading.columns = false;
+          }
+        };
+        $.ajax(request);
+      }
+      return cas(server_type, database, table);
+    }
+  };
+})($);

+ 35 - 0
apps/sqoop/static/js/sqoop.connections.js

@@ -65,6 +65,41 @@ var connections = (function($) {
         });
         return connection_string;
       });
+      self.jdbcDriver = ko.computed(function() {
+        var jdbc_driver = null;
+        $.each(self.connector(), function(index, form) {
+          if (form.name() == 'connection') {
+            $.each(form.inputs(), function(index, input) {
+              if (input.name() == 'connection.jdbcDriver') {
+                jdbc_driver = input.value();
+              }
+            });
+          }
+        });
+        return jdbc_driver;
+      });
+      self.database = ko.computed(function() {
+        var pattern = null;
+        switch (self.jdbcDriver()) {
+          case 'com.mysql.jdbc.Driver':
+          pattern = /jdbc:mysql:\/\/.*?\/(.*)/;
+          break;
+          case 'org.postgresql.Driver':
+          pattern = /jdbc:postgresql:\/\/.*?\/(.*)/;
+          break;
+          case 'oracle.jdbc.OracleDriver':
+          pattern = /jdbc:oracle:thin:@.*?:.*?:(.*)/;
+          break;
+        }
+        if (pattern) {
+          var res = self.connectionString().match(pattern);
+          if (res) {
+            return res[1];
+          } else {
+            return null;
+          }
+        }
+      });
     },
     'map': function() {
       var self = this;

+ 27 - 13
apps/sqoop/static/js/sqoop.jobs.js

@@ -127,24 +127,24 @@ var jobs = (function($) {
         return (output_directory) ? '/filebrowser/view' + output_directory : null;
       });
       self.inputDirectoryFilebrowserURL = ko.computed(function() {
-          var input_directory = null;
-          $.each(self.framework(), function(index, form) {
-            if (form.name() == 'input') {
-              $.each(form.inputs(), function(index, input) {
-                if (input.name() == 'input.inputDirectory') {
-                  input_directory = input.value();
-                }
-              });
-            }
-          });
-          return (input_directory) ? '/filebrowser/view' + input_directory : null;
+        var input_directory = null;
+        $.each(self.framework(), function(index, form) {
+          if (form.name() == 'input') {
+            $.each(form.inputs(), function(index, input) {
+              if (input.name() == 'input.inputDirectory') {
+                input_directory = input.value();
+              }
+            });
+          }
         });
+        return (input_directory) ? '/filebrowser/view' + input_directory : null;
+      });
       self.storageType = ko.computed(function() {
         var storage_type = null;
         $.each(self.framework(), function(index, form) {
-    	  if (form.name() == 'input') {
+          if (form.name() == 'input') {
             storage_type = 'HDFS'; // Hardcoded for now
-    	  } else if (form.name() == 'output') {
+          } else if (form.name() == 'output') {
             $.each(form.inputs(), function(index, input) {
               if (input.name() == 'output.storageType') {
                 storage_type = input.value();
@@ -154,6 +154,20 @@ var jobs = (function($) {
         });
         return storage_type;
       });
+      self.table = ko.computed(function() {
+        var table = null;
+        $.each(self.connector(), function(index, form) {
+          if (form.name() == 'table') {
+            console.log(form.name());
+            $.each(form.inputs(), function(index, input) {
+              if (input.name() == 'table.tableName') {
+                table = input.value();
+              }
+            });
+          }
+        });
+        return table;
+      });
 
       self.runningInterval = 0;
     },

+ 1 - 0
apps/sqoop/static/js/sqoop.js

@@ -41,6 +41,7 @@ function showSection(mainSection, section) {
     $(".section").hide();
     $("#" + section).show();
     highlightMenu(section);
+    $(document).trigger('shown_section', section);
   }
 }