Browse Source

HUE-7667 [core] Fix http client pool by correctly mounting Transport Adapter to a prefix.

Prakash Ranade 8 years ago
parent
commit
fdda567
1 changed files with 19 additions and 13 deletions
  1. 19 13
      desktop/core/src/desktop/lib/rest/http_client.py

+ 19 - 13
desktop/core/src/desktop/lib/rest/http_client.py

@@ -19,6 +19,7 @@ import posixpath
 import requests
 import requests
 import threading
 import threading
 import urllib
 import urllib
+from urlparse import urlparse
 
 
 from django.utils.encoding import iri_to_uri, smart_str
 from django.utils.encoding import iri_to_uri, smart_str
 from django.utils.http import urlencode
 from django.utils.http import urlencode
@@ -36,20 +37,19 @@ __docformat__ = "epytext"
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
-CACHE_SESSION = None
+CACHE_SESSION = {}
 CACHE_SESSION_LOCK = threading.Lock()
 CACHE_SESSION_LOCK = threading.Lock()
 
 
-def get_request_session():
-  global CACHE_SESSION
-  if CACHE_SESSION is None:
-    CACHE_SESSION_LOCK.acquire()
-    try:
-      if CACHE_SESSION is None:
-        CACHE_SESSION = requests.Session()
-        CACHE_SESSION.mount('http://', requests.adapters.HTTPAdapter(pool_connections=conf.CHERRYPY_SERVER_THREADS.get(), pool_maxsize=conf.CHERRYPY_SERVER_THREADS.get()))
-        CACHE_SESSION.mount('https://', requests.adapters.HTTPAdapter(pool_connections=conf.CHERRYPY_SERVER_THREADS.get(), pool_maxsize=conf.CHERRYPY_SERVER_THREADS.get()))
-    finally:
-      CACHE_SESSION_LOCK.release()
+def get_request_session(url, logger):
+  global CACHE_SESSION, CACHE_SESSION_LOCK
+
+  if CACHE_SESSION.get(url) is None:
+    with CACHE_SESSION_LOCK:
+      CACHE_SESSION[url] = requests.Session()
+      logger.debug("Setting request Session")
+      CACHE_SESSION[url].mount(url, requests.adapters.HTTPAdapter(pool_connections=conf.CHERRYPY_SERVER_THREADS.get(), pool_maxsize=conf.CHERRYPY_SERVER_THREADS.get()))
+      logger.debug("Setting session adapter for %s" % url)
+
   return CACHE_SESSION
   return CACHE_SESSION
 
 
 class RestException(Exception):
 class RestException(Exception):
@@ -103,9 +103,15 @@ class HttpClient(object):
     self._base_url = base_url.rstrip('/')
     self._base_url = base_url.rstrip('/')
     self._exc_class = exc_class or RestException
     self._exc_class = exc_class or RestException
     self._logger = logger or LOG
     self._logger = logger or LOG
-    self._session = get_request_session()
+    self._short_url = self._extract_netloc(self._base_url)
+    self._session = get_request_session(self._short_url, self._logger).get(self._short_url)
     self._cookies = None
     self._cookies = None
 
 
+  def _extract_netloc(self, base_url):
+    parsed_uri = urlparse(base_url)
+    short_url = '%(scheme)s://%(netloc)s' % {'scheme': parsed_uri.scheme, 'netloc': parsed_uri.netloc}
+    return short_url
+
   def set_kerberos_auth(self):
   def set_kerberos_auth(self):
     """Set up kerberos auth for the client, based on the current ticket."""
     """Set up kerberos auth for the client, based on the current ticket."""
     mutual_auth = conf.KERBEROS.MUTUAL_AUTHENTICATION.get().upper()
     mutual_auth = conf.KERBEROS.MUTUAL_AUTHENTICATION.get().upper()