瀏覽代碼

HUE-4653 [core] Build a HTTP Client pool

Prakash Ranade 9 年之前
父節點
當前提交
9bbc09f
共有 2 個文件被更改,包括 35 次插入1 次删除
  1. 18 0
      apps/oozie/src/oozie/tests.py
  2. 17 1
      desktop/core/src/desktop/lib/rest/http_client.py

+ 18 - 0
apps/oozie/src/oozie/tests.py

@@ -36,6 +36,7 @@ from django.core.urlresolvers import reverse
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_permission, add_to_group, reformat_json, reformat_xml
 from desktop.models import Document, Document2
+import desktop.views as views
 
 from hadoop import cluster as originalCluster
 from hadoop.pseudo_hdfs4 import is_live_cluster
@@ -3282,6 +3283,23 @@ my_prop_not_filtered=10
     finally:
       finish()
 
+  def test_httppool(self):
+    # With http pool the http connection is reused and so new connection count is 0
+    superuser_client = make_logged_in_client(is_superuser=True)
+    start_log = "--START HTTP POOL TEST--"
+    LOG.warn(start_log)
+    superuser_client.get(reverse('oozie:list_oozie_workflows'))
+    superuser_client.get(reverse('oozie:list_oozie_workflows') + "?format=json")
+    superuser_client.get(reverse('oozie:list_oozie_workflows') + "?format=json&status=RUNNING&status=PREP&status=SUSPENDED")
+    superuser_client.get(reverse('oozie:list_oozie_workflows') + "?format=json&status=KILLED&status=FAILED")
+    end_log = "--END HTTP POOL TEST--"
+    LOG.warn(end_log)
+    response = superuser_client.get(reverse(views.log_view))
+
+    s1 = response._container[0].index(start_log)
+    e1 = response._container[0].index(end_log)
+    c1 = response._container[0][e1:s1].count('Starting new HTTP')
+    assert_equal(c1, 0)
 
 class TestDashboard(OozieMockBase):
 

+ 17 - 1
desktop/core/src/desktop/lib/rest/http_client.py

@@ -17,6 +17,7 @@
 import logging
 import posixpath
 import requests
+import threading
 import urllib
 
 from django.utils.encoding import iri_to_uri, smart_str
@@ -34,6 +35,21 @@ __docformat__ = "epytext"
 
 LOG = logging.getLogger(__name__)
 
+CACHE_SESSION = None
+CACHE_SESSION_LOCK = threading.Lock()
+
+def get_request_session():
+  global CACHE_SESSION
+  if CACHE_SESSION is None:
+    CACHE_SESSION_LOCK.acquire()
+    try:
+      if CACHE_SESSION is None:
+        CACHE_SESSION = requests.Session()
+        CACHE_SESSION.mount('http://', requests.adapters.HTTPAdapter(pool_connections=conf.CHERRYPY_SERVER_THREADS.get(), pool_maxsize=conf.CHERRYPY_SERVER_THREADS.get()))
+        CACHE_SESSION.mount('https://', requests.adapters.HTTPAdapter(pool_connections=conf.CHERRYPY_SERVER_THREADS.get(), pool_maxsize=conf.CHERRYPY_SERVER_THREADS.get()))
+    finally:
+      CACHE_SESSION_LOCK.release()
+  return CACHE_SESSION
 
 class RestException(Exception):
   """
@@ -86,7 +102,7 @@ class HttpClient(object):
     self._base_url = base_url.rstrip('/')
     self._exc_class = exc_class or RestException
     self._logger = logger or LOG
-    self._session = requests.Session()
+    self._session = get_request_session()
     self._cookies = None
 
   def set_kerberos_auth(self):