|
@@ -18,6 +18,7 @@
|
|
|
import logging
|
|
import logging
|
|
|
import threading
|
|
import threading
|
|
|
import time
|
|
import time
|
|
|
|
|
+import random
|
|
|
|
|
|
|
|
from django.utils.translation import ugettext as _
|
|
from django.utils.translation import ugettext as _
|
|
|
|
|
|
|
@@ -32,24 +33,26 @@ LOG = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
API_CACHE = None
|
|
API_CACHE = None
|
|
|
API_CACHE_LOCK = threading.Lock()
|
|
API_CACHE_LOCK = threading.Lock()
|
|
|
|
|
+MAX_RETRIES = 15
|
|
|
|
|
|
|
|
|
|
|
|
|
def ha_error_handler(func):
|
|
def ha_error_handler(func):
|
|
|
def decorator(*args, **kwargs):
|
|
def decorator(*args, **kwargs):
|
|
|
- retries = 15
|
|
|
|
|
|
|
+ retries = 0
|
|
|
|
|
+ seed = random.random()
|
|
|
|
|
|
|
|
- while retries > 0:
|
|
|
|
|
|
|
+ while retries <= MAX_RETRIES:
|
|
|
try:
|
|
try:
|
|
|
return func(*args, **kwargs)
|
|
return func(*args, **kwargs)
|
|
|
except StructuredThriftTransportException, e:
|
|
except StructuredThriftTransportException, e:
|
|
|
- retries -= 1
|
|
|
|
|
- if not is_ha_enabled() or retries == 0:
|
|
|
|
|
|
|
+ if not is_ha_enabled() or retries == MAX_RETRIES:
|
|
|
raise PopupException(_('Failed to retry connecting to an available Sentry server.'), detail=e)
|
|
raise PopupException(_('Failed to retry connecting to an available Sentry server.'), detail=e)
|
|
|
else:
|
|
else:
|
|
|
LOG.info('Could not connect to Sentry server %s, attempting to fetch next available client.' % args[0].client.host)
|
|
LOG.info('Could not connect to Sentry server %s, attempting to fetch next available client.' % args[0].client.host)
|
|
|
time.sleep(1)
|
|
time.sleep(1)
|
|
|
- args[0].client = get_cached_client(args[0].client.username, force_reset=True)
|
|
|
|
|
- LOG.info('Picked %s' % args[0].client)
|
|
|
|
|
|
|
+ args[0].client = get_cached_client(args[0].client.username, retries=retries, seed=seed)
|
|
|
|
|
+ LOG.info('Picked %s at attempt %s' % (args[0].client, retries))
|
|
|
|
|
+ retries += 1
|
|
|
except SentryException, e:
|
|
except SentryException, e:
|
|
|
raise e
|
|
raise e
|
|
|
except Exception, e:
|
|
except Exception, e:
|
|
@@ -67,8 +70,9 @@ def get_api(user, component):
|
|
|
return SentryApi(client)
|
|
return SentryApi(client)
|
|
|
|
|
|
|
|
|
|
|
|
|
-def get_cached_client(username, component=None, force_reset=False):
|
|
|
|
|
|
|
+def get_cached_client(username, component=None, retries=0, seed=None):
|
|
|
exempt_host = None
|
|
exempt_host = None
|
|
|
|
|
+ force_reset = retries > 0
|
|
|
|
|
|
|
|
global API_CACHE
|
|
global API_CACHE
|
|
|
if force_reset and API_CACHE is not None:
|
|
if force_reset and API_CACHE is not None:
|
|
@@ -80,7 +84,7 @@ def get_cached_client(username, component=None, force_reset=False):
|
|
|
if API_CACHE is None:
|
|
if API_CACHE is None:
|
|
|
API_CACHE_LOCK.acquire()
|
|
API_CACHE_LOCK.acquire()
|
|
|
try:
|
|
try:
|
|
|
- API_CACHE = get_sentry_client(username, SentryClient, exempt_host=exempt_host, component=component)
|
|
|
|
|
|
|
+ API_CACHE = get_sentry_client(username, SentryClient, exempt_host=exempt_host, component=component, retries=retries, seed=seed)
|
|
|
LOG.info("Setting cached Sentry client to host: %s" % API_CACHE.host)
|
|
LOG.info("Setting cached Sentry client to host: %s" % API_CACHE.host)
|
|
|
finally:
|
|
finally:
|
|
|
API_CACHE_LOCK.release()
|
|
API_CACHE_LOCK.release()
|