瀏覽代碼

HUE-2320 [impala] Invalidate medata of new or deleted tables

Happens when clicking on the refresh button
Romain Rigaux 11 年之前
父節點
當前提交
ea27847d78

+ 12 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -27,11 +27,10 @@ from beeswax import hive_site
 from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT,\
   BROWSE_PARTITIONED_TABLE_LIMIT
 from beeswax.design import hql_query
-from beeswax.models import QueryHistory, HIVE_SERVER2, BEESWAX, QUERY_TYPES
+from beeswax.models import QueryHistory, QUERY_TYPES
 
 from filebrowser.views import location_to_url
 from desktop.lib.django_util import format_preserving_redirect
-from desktop.lib.exceptions_renderable import PopupException
 
 
 LOG = logging.getLogger(__name__)
@@ -238,6 +237,17 @@ class HiveServer2Dbms(object):
     return self.execute_query(query, design)
 
 
+  def invalidate_tables(self, database, tables):
+    for table in tables:
+      hql = "INVALIDATE METADATA %s.%s" % (database, table,)        
+      query = hql_query(hql, database, query_type=QUERY_TYPES[1])
+
+      handle = self.execute_and_wait(query, timeout_sec=5.0)
+
+      if handle:
+        self.close(handle)
+
+
   def drop_database(self, database):
     return self.execute_statement("DROP DATABASE `%s`" % database)
 

+ 42 - 29
apps/beeswax/src/beeswax/templates/execute.mako

@@ -42,8 +42,12 @@ ${layout.menubar(section='query')}
     <div class="tab-pane active" id="navigatorTab">
       <div class="card card-small card-tab">
         <div class="card-body" style="margin-top: 0">
-          <a href="#" title="${_('Double click on a table name or field to insert it in the editor')}" rel="tooltip" data-placement="top" class="pull-right" style="margin:3px; margin-top:7px"><i class="fa fa-question-circle"></i></a>
-          <a id="refreshNavigator" href="#" title="${_('Manually refresh the table list')}" rel="tooltip" data-placement="top" class="pull-right" style="margin:3px; margin-top:7px"><i class="fa fa-refresh"></i></a>
+          <a href="#" title="${_('Double click on a table name or field to insert it in the editor')}" rel="tooltip" data-placement="top" class="pull-right" style="margin:3px; margin-top:7px">
+            <i class="fa fa-question-circle"></i>
+          </a>
+          <a id="refreshNavigator" href="#" title="${_('Manually refresh the table list')}" rel="tooltip" data-placement="top" class="pull-right" style="margin:3px; margin-top:7px">
+            <i class="fa fa-refresh"></i>
+          </a>
           <ul class="nav nav-list" style="border: none; padding: 0; background-color: #FFF">
             <li class="nav-header">${_('database')}</li>
           </ul>
@@ -1198,33 +1202,6 @@ $(document).ready(function () {
     }
   }
 
-  % if app_name == 'impala':
-    syncWithHive = function () {
-      // sync tables with Hive
-      hac_jsoncalls({
-        autocompleteBaseURL: "${ autocomplete_base_url_hive | n,unicode }",
-        database: viewModel.database(),
-        onDataReceived: function (data) {
-          if (data.tables) {
-            var _hiveTables = data.tables;
-            hac_getTables(viewModel.database(), function (data) {  //preload tables for the default db
-              var _impalaTables = data.split(" ");
-              var _diff = {
-                added: _hiveTables.diff(_impalaTables),
-                removed: _impalaTables.diff(_hiveTables)
-              }
-              console.log("Tables diff")
-              console.log(_diff);
-            });
-          }
-        }
-      });
-    }
-    if (viewModel.database()) {
-      syncWithHive();
-    }
-  % endif
-
   $("#expandResults").on("click", function(){
     if ($(this).find("i").hasClass("fa-expand")){
       $(this).find("i").removeClass("fa-expand").addClass("fa-compress");
@@ -1238,6 +1215,42 @@ $(document).ready(function () {
   });
 
   $("#refreshNavigator").on("click", function () {
+    % if app_name == 'impala':
+      syncWithHive = function () {
+        // Diff Hive / Impala metastore and invalidate out of sync tables
+        hac_jsoncalls({
+          autocompleteBaseURL: "${ autocomplete_base_url_hive | n,unicode }",
+          database: viewModel.database(),
+          sync: true,
+          onDataReceived: function (data) {
+            if (data.tables) {
+              var _hiveTables = data.tables;
+              hac_getTables(viewModel.database(), function (data) {
+                var _impalaTables = data.split(" ");
+                $.ajax({
+                  type: "POST",
+                  url: "${ url('impala:refresh_tables') }",
+                  data: {
+                    database: ko.mapping.toJSON(viewModel.database()),
+                    added: ko.mapping.toJSON(_hiveTables.diff(_impalaTables)),
+                    removed: ko.mapping.toJSON(_impalaTables.diff(_hiveTables)),
+                  },
+                  success: function (data) {
+                    if (data.status != 0) {
+                      $(document).trigger("error", data.message);
+                    }
+                  }
+                });
+              });
+            }
+          }
+        });
+      }
+    
+      syncWithHive();
+
+    % endif
+   
     resetNavigator();
   });
 

+ 13 - 19
apps/beeswax/static/js/autocomplete.utils.js

@@ -15,27 +15,21 @@
 // limitations under the License.
 
 function hac_jsoncalls(options) {
-  if (typeof HIVE_AUTOCOMPLETE_BASE_URL != "undefined" || typeof options.autocompleteBaseURL != "undefined") {
-    var _baseURL = typeof options.autocompleteBaseURL != "undefined" ? options.autocompleteBaseURL : HIVE_AUTOCOMPLETE_BASE_URL;
-    if (options.database == null) {
-      $.getJSON(_baseURL, options.onDataReceived);
-    }
-    if (options.database != null) {
-      if (options.table != null) {
-        $.getJSON(_baseURL + options.database + "/" + options.table, options.onDataReceived);
-      }
-      else {
-        $.getJSON(_baseURL + options.database + "/", options.onDataReceived);
-      }
-    }
+  var _url = typeof options.autocompleteBaseURL != "undefined" ? options.autocompleteBaseURL : HIVE_AUTOCOMPLETE_BASE_URL;
+    
+  if (options.database != null) {
+    _url += options.database
   }
-  else {
-    try {
-      console.error("You need to specify a HIVE_AUTOCOMPLETE_BASE_URL to use the autocomplete")
-    }
-    catch (e) {
-    }
+  if (options.table != null) {
+    _url += "/" + options.table
   }
+  
+  $.ajax({
+    type: "GET",
+    url: _url,
+    success: options.onDataReceived,
+    async: options.sync == "undefined"
+  });
 }
 
 function hac_hasExpired(timestamp){

+ 2 - 0
apps/impala/src/impala/urls.py

@@ -21,6 +21,8 @@ from beeswax.urls import urlpatterns as beeswax_urls
 
 
 urlpatterns = patterns('impala.views',
+  url(r'^api/refresh_tables$', 'refresh_tables', name='refresh_tables'),
+
   url(r'^dashboard/$', 'dashboard', name='dashboard'),
   url(r'^dashboard/query', 'query', name='query'),
   

+ 28 - 1
apps/impala/src/impala/views.py

@@ -24,7 +24,9 @@ import json
 
 from django.http import HttpResponse
 
+from desktop.context_processors import get_app_name
 from desktop.lib.django_util import render
+
 from beeswax.design import hql_query
 from beeswax.server import dbms
 from beeswax.server.dbms import get_query_server_config
@@ -35,6 +37,31 @@ from impala.models import Dashboard, Controller
 LOG = logging.getLogger(__name__)
 
 
+def refresh_tables(request):
+  app_name = get_app_name(request)
+  query_server = get_query_server_config(app_name)  
+  db = dbms.get(request.user, query_server=query_server)
+  
+  response = {'status': -1, 'message': ''}
+  
+  if request.method == "POST":
+    try:
+      database = json.loads(request.POST['database'])
+      added = json.loads(request.POST['added'])
+      removed = json.loads(request.POST['removed'])
+      
+      db.invalidate_tables(database, added + removed)
+      
+      response['status'] = 0
+    except Exception, e:
+      response['message'] = str(e)    
+  
+  return HttpResponse(json.dumps(response), mimetype="application/json")
+
+
+# Dashboard below
+
+
 def dashboard(request):
   dashboard = Dashboard()
 
@@ -51,7 +78,7 @@ def query(request):
     'data': {}
   }
   
-  dashboard = json.loads(request.POST['dashboard'])  
+  dashboard = json.loads(request.POST['dashboard'])
   fqs = json.loads(request.POST['query'])['fqs']
 
   database = dashboard['properties'][0]['database']