Эх сурвалжийг харах

HUE-1298 [impala] Multiple Impalad support

Made the DBMS.get method a factory that reuses instances on a per user per interface basis.
Impala sessions close when connectivity is lost.
Load balancing multiple impalad's with a persistent connection
enables Hue to communicate with the same impalad for as long as Hue is running
and the connection is alive.
Abraham Elmahrek 11 жил өмнө
parent
commit
f27cc76

+ 17 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -16,10 +16,10 @@
 # limitations under the License.
 
 import logging
+import threading
 import time
 
 from django.core.urlresolvers import reverse
-from django.shortcuts import redirect
 from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 
@@ -36,15 +36,30 @@ from desktop.lib.exceptions_renderable import PopupException
 
 LOG = logging.getLogger(__name__)
 
+DBMS_CACHE = {}
+DBMS_CACHE_LOCK = threading.Lock()
+
 
 def get(user, query_server=None):
+  global DBMS_CACHE
+  global DBMS_CACHE_LOCK
+
   # Avoid circular dependency
   from beeswax.server.hive_server2_lib import HiveServerClientCompatible, HiveServerClient
 
   if query_server is None:
     query_server = get_query_server_config()
 
-  return HiveServer2Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
+  DBMS_CACHE_LOCK.acquire()
+  try:
+    DBMS_CACHE.setdefault(user.username, {})
+
+    if query_server['server_name'] not in DBMS_CACHE[user.username]:
+      DBMS_CACHE[user.username][query_server['server_name']] = HiveServer2Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
+
+    return DBMS_CACHE[user.username][query_server['server_name']]
+  finally:
+    DBMS_CACHE_LOCK.release()
 
 
 def get_query_server_config(name='beeswax', server=None):

+ 3 - 2
apps/beeswax/src/beeswax/templates/execute.mako

@@ -16,7 +16,8 @@
 <%!
   from desktop.lib.django_util import extract_field_data
   from desktop.views import commonheader, commonfooter
-  from beeswax.conf import CLOSE_QUERIES
+  from beeswax import conf as beeswax_conf
+  from impala import conf as impala_conf
   from django.utils.translation import ugettext as _
 %>
 
@@ -2270,7 +2271,7 @@ $(document).ready(function () {
 });
 % endif
 
-% if CLOSE_QUERIES.get() or app_name == 'impala':
+% if ( app_name == 'beeswax' and beeswax_conf.CLOSE_QUERIES.get() ) or ( app_name == 'impala' and impala_conf.CLOSE_QUERIES.get() ):
 $(document).ready(function () {
   $(document).on('explain.query', function() {
     viewModel.closeQuery();

+ 8 - 0
apps/impala/src/impala/conf.py

@@ -55,6 +55,14 @@ QUERYCACHE_ROWS=Config(
   type=int,
   default=50000)
 
+CLOSE_QUERIES = Config(
+  key="close_queries",
+  help=_t("Hue will try to close the Impala query when the user leaves the editor page. "
+          "This will free all the query resources in Impala, but also make its results inaccessible."),
+  type=coerce_bool,
+  default=True
+)
+
 
 def config_validator(user):
   # dbms is dependent on beeswax.conf (this file)

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -702,6 +702,10 @@
   # Set to 0 for disabling the option and backward compatibility.
   ## querycache_rows=50000
 
+  # Hue will try to close the Impala query when the user leaves the editor page.
+  # This will free all the query resources in Impala, but also make its results inaccessible.
+  ## close_queries=true
+
 
 ###########################################################################
 # Settings to configure Hbase

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -743,6 +743,10 @@
   # Set to 0 for disabling the option and backward compatibility.
   ## querycache_rows=50000
 
+  # Hue will try to close the Impala query when the user leaves the editor page.
+  # This will free all the query resources in Impala, but also make its results inaccessible.
+  ## close_queries=true
+
 
 ###########################################################################
 # Settings to configure Hbase