Browse Source

[metrics] Fix lost connection when metrics query database (#3852)

Ying Chen 1 year ago
parent
commit
9111b916f7
2 changed files with 58 additions and 15 deletions
  1. 26 4
      apps/useradmin/src/useradmin/metrics.py
  2. 32 11
      desktop/core/src/desktop/metrics.py

+ 26 - 4
apps/useradmin/src/useradmin/metrics.py

@@ -15,8 +15,10 @@
 # limitations under the License.
 
 import logging
-
 from datetime import datetime, timedelta
+
+from django.db import connection
+from django.db.utils import OperationalError
 from prometheus_client import Gauge
 
 from desktop.lib.metrics import global_registry
@@ -24,6 +26,7 @@ from desktop.lib.security_util import get_localhost_name
 
 LOG = logging.getLogger()
 
+
 def active_users():
   from useradmin.models import UserProfile
   try:
@@ -32,11 +35,21 @@ def active_users():
         first_login=False,
         hostname__isnull=False
     ).count()
-  except:
+  except OperationalError as oe:
+    LOG.debug('active_users recovering from %s' % str(oe))
+    connection.close()
+    connection.connect()
+    count = UserProfile.objects.filter(
+        last_activity__gt=datetime.now() - timedelta(hours=1),
+        first_login=False,
+        hostname__isnull=False
+    ).count()
+  except Exception as e:
     LOG.exception('Could not get active_users')
     count = 0
   return count
 
+
 global_registry().gauge_callback(
     name='users.active.total',
     callback=active_users,
@@ -48,15 +61,24 @@ global_registry().gauge_callback(
 prometheus_active_users = Gauge('hue_active_users', 'Hue Active Users in All Instances')
 prometheus_active_users.set_function(active_users)
 
+
 def active_users_per_instance():
   from useradmin.models import UserProfile
   try:
-    count = UserProfile.objects.filter(last_activity__gt=datetime.now() - timedelta(hours=1), hostname=get_localhost_name()).count()
-  except:
+    count = UserProfile.objects.filter(last_activity__gt=datetime.now() - timedelta(hours=1),
+                                       hostname=get_localhost_name()).count()
+  except OperationalError as oe:
+    LOG.debug('active_users_per_instance recovering from %s' % str(oe))
+    connection.close()
+    connection.connect()
+    count = UserProfile.objects.filter(last_activity__gt=datetime.now() - timedelta(hours=1),
+                                       hostname=get_localhost_name()).count()
+  except Exception as e:
     LOG.exception('Could not get active_users per instance')
     count = 0
   return count
 
+
 global_registry().gauge_callback(
     name='users.active',
     callback=active_users_per_instance,

+ 32 - 11
desktop/core/src/desktop/metrics.py

@@ -16,22 +16,23 @@
 
 from __future__ import absolute_import
 
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
 import gc
 import logging
-import multiprocessing
 import threading
-
+import multiprocessing
+from builtins import range
 from datetime import datetime, timedelta
-from prometheus_client import Gauge, REGISTRY
 
-from useradmin.models import User
+from django.db import connection
+from django.db.utils import OperationalError
+from future import standard_library
+from prometheus_client import REGISTRY, Gauge
 
 from desktop.conf import ENABLE_PROMETHEUS
 from desktop.lib.metrics import global_registry
+from useradmin.models import User
 
+standard_library.install_aliases()
 
 LOG = logging.getLogger()
 
@@ -49,7 +50,9 @@ if ENABLE_PROMETHEUS.get():
   django_collectors = set()
   django_metrics_names = [
       name
-      for name in REGISTRY._names_to_collectors.keys() if name.startswith('django_') and not name.startswith(ALLOWED_DJANGO_PROMETHEUS_METRICS)
+      for name in REGISTRY._names_to_collectors.keys()
+        if name.startswith('django_')
+           and not name.startswith(ALLOWED_DJANGO_PROMETHEUS_METRICS)
   ]
 
   for metric_name in django_metrics_names:
@@ -141,14 +144,21 @@ response_time = global_registry().timer(
 
 # ------------------------------------------------------------------------------
 
+
 def user_count():
   users = 0
   try:
     users = User.objects.count()
-  except:
+  except OperationalError as oe:
+    LOG.debug('user_count recovering from %s' % str(oe))
+    connection.close()
+    connection.connect()
+    users = User.objects.count()
+  except Exception as e:
     LOG.exception('Metrics: Failed to get number of user accounts')
   return users
 
+
 user_count = global_registry().gauge_callback(
     name='users',
     callback=user_count,
@@ -188,19 +198,30 @@ spnego_authentication_time = global_registry().timer(
 
 # ------------------------------------------------------------------------------
 
+
 def num_of_queries():
-  from desktop.models import Document2 # Avoid circular dependency
+  from desktop.models import Document2  # Avoid circular dependency
   try:
     count = Document2.objects.filter(
       type__istartswith='query-',
       is_history=True,
       last_modified__gt=datetime.now() - timedelta(minutes=10)
     ).count()
-  except:
+  except OperationalError as oe:
+    LOG.debug('num_of_queries recovering from %s' % str(oe))
+    connection.close()
+    connection.connect()
+    count = Document2.objects.filter(
+      type__istartswith='query-',
+      is_history=True,
+      last_modified__gt=datetime.now() - timedelta(minutes=10)
+    ).count()
+  except Exception as e:
     LOG.exception('Could not get num_of_queries')
     count = 0
   return count
 
+
 global_registry().gauge_callback(
     name='queries.number',
     callback=num_of_queries,