Răsfoiți Sursa

HUE-1298 [impala] Multiple Impalad support

Made the DBMS.get method a factory that reuses instances on a per user per interface basis.
Impala sessions close when connectivity is lost.
Load balancing multiple impalad's with a persistent connection
enables Hue to communicate with the same impalad for as long as Hue is running
and the connection is alive.
Abraham Elmahrek 11 ani în urmă
părinte
comite
f27cc76

+ 17 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -16,10 +16,10 @@
 # limitations under the License.
 
 import logging
+import threading
 import time
 
 from django.core.urlresolvers import reverse
-from django.shortcuts import redirect
 from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 
@@ -36,15 +36,30 @@ from desktop.lib.exceptions_renderable import PopupException
 
 LOG = logging.getLogger(__name__)
 
+DBMS_CACHE = {}
+DBMS_CACHE_LOCK = threading.Lock()
+
 
 def get(user, query_server=None):
+  global DBMS_CACHE
+  global DBMS_CACHE_LOCK
+
   # Avoid circular dependency
   from beeswax.server.hive_server2_lib import HiveServerClientCompatible, HiveServerClient
 
   if query_server is None:
     query_server = get_query_server_config()
 
-  return HiveServer2Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
+  DBMS_CACHE_LOCK.acquire()
+  try:
+    DBMS_CACHE.setdefault(user.username, {})
+
+    if query_server['server_name'] not in DBMS_CACHE[user.username]:
+      DBMS_CACHE[user.username][query_server['server_name']] = HiveServer2Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
+
+    return DBMS_CACHE[user.username][query_server['server_name']]
+  finally:
+    DBMS_CACHE_LOCK.release()
 
 
 def get_query_server_config(name='beeswax', server=None):

+ 3 - 2
apps/beeswax/src/beeswax/templates/execute.mako

@@ -16,7 +16,8 @@
 <%!
   from desktop.lib.django_util import extract_field_data
   from desktop.views import commonheader, commonfooter
-  from beeswax.conf import CLOSE_QUERIES
+  from beeswax import conf as beeswax_conf
+  from impala import conf as impala_conf
   from django.utils.translation import ugettext as _
 %>
 
@@ -2270,7 +2271,7 @@ $(document).ready(function () {
 });
 % endif
 
-% if CLOSE_QUERIES.get() or app_name == 'impala':
+% if ( app_name == 'beeswax' and beeswax_conf.CLOSE_QUERIES.get() ) or ( app_name == 'impala' and impala_conf.CLOSE_QUERIES.get() ):
 $(document).ready(function () {
   $(document).on('explain.query', function() {
     viewModel.closeQuery();

+ 8 - 0
apps/impala/src/impala/conf.py

@@ -55,6 +55,14 @@ QUERYCACHE_ROWS=Config(
   type=int,
   default=50000)
 
+CLOSE_QUERIES = Config(
+  key="close_queries",
+  help=_t("Hue will try to close the Impala query when the user leaves the editor page. "
+          "This will free all the query resources in Impala, but also make its results inaccessible."),
+  type=coerce_bool,
+  default=True
+)
+
 
 def config_validator(user):
   # dbms is dependent on beeswax.conf (this file)

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -702,6 +702,10 @@
   # Set to 0 for disabling the option and backward compatibility.
   ## querycache_rows=50000
 
+  # Hue will try to close the Impala query when the user leaves the editor page.
+  # This will free all the query resources in Impala, but also make its results inaccessible.
+  ## close_queries=true
+
 
 ###########################################################################
 # Settings to configure Hbase

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -743,6 +743,10 @@
   # Set to 0 for disabling the option and backward compatibility.
   ## querycache_rows=50000
 
+  # Hue will try to close the Impala query when the user leaves the editor page.
+  # This will free all the query resources in Impala, but also make its results inaccessible.
+  ## close_queries=true
+
 
 ###########################################################################
 # Settings to configure Hbase