Эх сурвалжийг харах

HUE-8882 [impala] Fix invalidate delta when hive is missing.

Jean-Francois Desjeans Gauthier 6 жил өмнө
parent
commit
a1355e7a5e

+ 22 - 9
apps/impala/src/impala/dbms.py

@@ -17,7 +17,7 @@
 
 import logging
 
-from desktop.lib.i18n import smart_str
+from django.utils.translation import ugettext as _
 
 from beeswax.design import hql_query
 from beeswax.models import QUERY_TYPES
@@ -26,7 +26,12 @@ from beeswax.server.dbms import HiveServer2Dbms, QueryServerException, QueryServ
   get_query_server_config as beeswax_query_server_config
 
 from desktop.conf import CLUSTER_ID
+from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.i18n import smart_str
+from desktop.models import Cluster, ClusterConfig
+
 from impala import conf
+from notebook.conf import get_ordered_interpreters
 
 
 LOG = logging.getLogger(__name__)
@@ -101,19 +106,26 @@ class ImpalaDbms(HiveServer2Dbms):
         query = hql_query(hql, query_type=QUERY_TYPES[1])
         handle = self.execute_and_wait(query, timeout_sec=10.0)
       elif table is None:
+        if not Cluster(self.client.user).get_app_config().get_hive_metastore_interpreters():
+          raise PopupException(_("Hive and HMS not configured. Please do a full refresh"))
         diff_tables = self._get_different_tables(database)
-        for table in diff_tables:
-          hql = "INVALIDATE METADATA `%s`.`%s`" % (database, table)
-          query = hql_query(hql, query_type=QUERY_TYPES[1])
-          handle = self.execute_and_wait(query, timeout_sec=10.0)
+        if len(diff_tables) > 10:
+          raise PopupException(_("Too many tables (%s) to invalidate. Please do a full refresh") % str(len(diff_tables)))
+        else:
+          for table in diff_tables:
+            hql = "INVALIDATE METADATA `%s`.`%s`" % (database, table)
+            query = hql_query(hql, query_type=QUERY_TYPES[1])
+            handle = self.execute_and_wait(query, timeout_sec=10.0)
       else:
         hql = "INVALIDATE METADATA `%s`.`%s`" % (database, table)
         query = hql_query(hql, query_type=QUERY_TYPES[1])
         handle = self.execute_and_wait(query, timeout_sec=10.0)
-    except QueryServerTimeoutException, e:
+    except QueryServerTimeoutException as e:
       # Allow timeout exceptions to propagate
       raise e
-    except Exception, e:
+    except PopupException as e:
+      raise e
+    except Exception as e:
       msg = 'Failed to invalidate `%s`: %s' % (database or 'databases', e)
       raise QueryServerException(msg)
     finally:
@@ -170,11 +182,12 @@ class ImpalaDbms(HiveServer2Dbms):
 
 
   def _get_beeswax_tables(self, database):
-    beeswax_query_server = dbms.get(user=self.client.user, query_server=beeswax_query_server_config(name='beeswax'))
+    interpreters = ['beeswax' if interpreter == 'hive' else interpreter for interpreter in Cluster(self.client.user).get_app_config().get_hive_metastore_interpreters()]
+    beeswax_query_server = dbms.get(user=self.client.user, query_server=beeswax_query_server_config(name=interpreters[0]))
     return beeswax_query_server.get_tables(database=database)
 
 
   def _get_different_tables(self, database):
     beeswax_tables = self._get_beeswax_tables(database)
     impala_tables = self.get_tables(database=database)
-    return set(beeswax_tables).symmetric_difference(impala_tables)
+    return set(beeswax_tables).symmetric_difference(impala_tables)

+ 25 - 1
apps/impala/src/impala/tests.py

@@ -20,13 +20,15 @@ import logging
 import re
 
 from nose.plugins.skip import SkipTest
-from nose.tools import assert_true, assert_equal, assert_false
+from nose.tools import assert_true, assert_equal, assert_false, assert_raises
+from mock import patch, Mock
 
 from django.contrib.auth.models import User
 from django.urls import reverse
 
 import desktop.conf as desktop_conf
 from desktop.lib.django_test_util import make_logged_in_client
+from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.test_utils import add_to_group
 from desktop.models import Document
 from hadoop.pseudo_hdfs4 import get_db_prefix, is_live_cluster
@@ -104,6 +106,27 @@ class TestMockedImpala:
       if impala_query is not None:
         impala_query.delete()
 
+  def test_invalidate(self):
+    with patch('impala.dbms.ImpalaDbms._get_different_tables') as get_different_tables:
+      with patch('desktop.models.ClusterConfig.get_hive_metastore_interpreters') as get_hive_metastore_interpreters:
+        ddms = ImpalaDbms(Mock(query_server={'server_name': ''}), None)
+        get_different_tables.return_value = ['customers']
+
+        get_hive_metastore_interpreters.return_value = []
+        assert_raises(PopupException, ddms.invalidate, 'default') # No hive/metastore configured
+
+        get_hive_metastore_interpreters.return_value = ['hive']
+        ddms.invalidate('default')
+        assert_true('customers' in ddms.client.query.call_args[0][0].hql_query) # diff of 1 table
+
+        get_different_tables.return_value = ['customers','','','','','','','','','','']
+        assert_raises(PopupException, ddms.invalidate, 'default') # diff of 11 tables. Limit is 10.
+
+        ddms.invalidate('default', 'customers')
+        assert_true('customers' in ddms.client.query.call_args[0][0].hql_query) # invalidate 1 table
+
+        ddms.invalidate()
+        assert_true('customers' not in ddms.client.query.call_args[0][0].hql_query) # Full invalidate
 
 class TestImpalaIntegration:
   integration = True
@@ -488,3 +511,4 @@ class TestImpalaDbms():
                  ('order_date', '`default`.`customers`.`orders`'))
     assert_equal(ImpalaDbms.get_nested_select('default', 'customers', 'orders', 'item/items/item/product_id'),
                  ('product_id', '`default`.`customers`.`orders`.`items`'))
+

+ 6 - 4
desktop/core/src/desktop/models.py

@@ -1551,10 +1551,7 @@ class Document2Permission(models.Model):
 
 
 def get_cluster_config(user):
-  cluster_type = Cluster(user).get_type()
-  cluster_config = ClusterConfig(user, cluster_type=cluster_type)
-
-  return cluster_config.get_config()
+  return Cluster(user).get_app_config().get_config()
 
 
 # Aka 'Atus'
@@ -1929,6 +1926,8 @@ class ClusterConfig():
     else:
       return None
 
+  def get_hive_metastore_interpreters(self):
+    return list(filter(lambda interpreter: interpreter == 'hive' or interpreter == 'hms', [interpreter['type'] for interpreter in get_ordered_interpreters(self.user)]))
 
 class Cluster():
 
@@ -1952,6 +1951,9 @@ class Cluster():
   def get_config(self, name):
     return self.clusters[name]
 
+  def get_app_config(self):
+    return ClusterConfig(self.user, cluster_type=self.get_type())
+
 
 def _get_apps(user, section=None):
   current_app = None