Pārlūkot izejas kodu

HUE-8805 [notebook] Add query execution counts of shared queries

Can take a User / Document or id in input.
Give various stats and an histogram of daily usage in a rolling 30 days

e.g.

Anaytics.user_stats(1)

[{'name': 'user', 'value': u'1 - romain', 'description': u'User info'}, {'values': 6284, 'name': 'query_executions', 'description': u'Query executions count'}, {'name': 'saved_queries_count', 'value': 61, 'description': u'Saved queries count'}, {'name': 'query_executions_30_days_count', 'value': 215, 'description': u'Query executions 30 days total'}, {'name': 'query_executions_30_days_histogram', 'value': <Document2QuerySet [{'c': 2, 'day': datetime.datetime(2019, 3, 18, 0, 0)}, {'c': 4, 'day': datetime.datetime(2019, 3, 19, 0, 0)}, {'c': 7, 'day': datetime.datetime(2019, 3, 21, 0, 0)}, {'c': 3, 'day': datetime.datetime(2019, 3, 22, 0, 0)}, {'c': 53, 'day': datetime.datetime(2019, 3, 26, 0, 0)}, {'c': 42, 'day': datetime.datetime(2019, 3, 27, 0, 0)}, {'c': 7, 'day': datetime.datetime(2019, 3, 28, 0, 0)}, {'c': 2, 'day': datetime.datetime(2019, 3, 29, 0, 0)}, {'c': 9, 'day': datetime.datetime(2019, 4, 1, 0, 0)}, {'c': 1, 'day': datetime.datetime(2019, 4, 2, 0, 0)}, {'c': 2, 'day': datetime.datetime(2019, 4, 3, 0, 0)}, {'c': 1, 'day': datetime.datetime(2019, 4, 4, 0, 0)}, {'c': 8, 'day': datetime.datetime(2019, 4, 5, 0, 0)}, {'c': 22, 'day': datetime.datetime(2019, 4, 10, 0, 0)}, {'c': 2, 'day': datetime.datetime(2019, 4, 11, 0, 0)}, {'c': 1, 'day': datetime.datetime(2019, 4, 14, 0, 0)}, {'c': 4, 'day': datetime.datetime(2019, 4, 15, 0, 0)}, {'c': 32, 'day': datetime.datetime(2019, 4, 16, 0, 0)}, {'c': 17, 'day': datetime.datetime(2019, 4, 17, 0, 0)}]>, 'description': u'Daily executions 30 days'}]

Anaytics.query_stats(54042)

[{'name': 'query', 'value': u'54042 - Sample: Customers', 'description': u'Query info'}, {'name': 'execution_count', 'value': 2, 'description': u'How many times executed'}, {'name': 'execution_count_shared', 'value': 2, 'description': u'Executions by others'}]
Romain Rigaux 6 gadi atpakaļ
vecāks
revīzija
c617453173

+ 54 - 13
desktop/libs/notebook/src/notebook/models.py

@@ -29,6 +29,7 @@ from django.contrib.auth.models import User
 from django.db.models import Count
 from django.db.models.functions import Trunc
 from django.utils.html import escape
+from django.utils.translation import ugettext as _
 
 from desktop.lib.i18n import smart_unicode
 from desktop.models import Document2
@@ -486,33 +487,73 @@ class Anaytics():
     stats.append(('Active users 30 days', User.objects.filter(last_login__gte=one_month).count()))
     stats.append(('Active users 90 days', User.objects.filter(last_login__gte=three_months).count()))
 
-    print(cls.user_stats(1))
-    print(cls.query_stats(69371))
-
     return stats
 
   @classmethod
-  def user_stats(cls, user_id):
+  def user_stats(cls, user_id, user=None):
     stats = []
     one_month = datetime.date.today() - timedelta(days=30)
 
-    user = User.objects.get(id=user_id)
+    user = User.objects.get(id=user_id) if user is None else user
     queries = Document2.objects.filter(owner__id=user_id, type__startswith='query-', is_trashed=False, is_managed=False)
 
-    stats.append(('User', '%s - %s' % (user_id, user.username)))
-    stats.append(('Executions', queries.filter(is_history=True, type__startswith='query-').count()))
-    stats.append(('Saved', queries.filter(is_history=False, type__startswith='query-').count()))
+    stats.append({
+      'name': 'user',
+      'value': '%s - %s' % (user_id, user.username),'description': _('User info')
+    })
+    query_executions = queries.filter(is_history=True, type__startswith='query-')
+    stats.append({
+      'name': 'query_executions',
+      'values': query_executions.count(),
+      'description': _('Query executions count')
+    })
+    stats.append({
+      'name': 'saved_queries_count',
+      'value': queries.filter(is_history=False, type__startswith='query-').count(),
+      'description': _('Saved queries count')
+    })
+    stats.append({
+      'name': 'query_executions_30_days_count',
+      'value': query_executions.filter(last_modified__gte=one_month).count(),
+      'description': _('Query executions 30 days total')
+    })
     last_month_daily = queries.filter(last_modified__gte=one_month).annotate(day=Trunc('last_modified', 'day')).values('day').annotate(c=Count('day')).values('day', 'c').order_by('day')
-    stats.append(('Daily executions 30 days', last_month_daily))
+    stats.append({
+      'name': 'query_executions_30_days_histogram',
+      'value': last_month_daily,
+      'description': _('Daily executions 30 days')
+    })
 
     return stats
 
   @classmethod
-  def query_stats(cls, query_id):
+  def query_stats(cls, query_id, query=None):
     stats = []
+    one_month = datetime.date.today() - timedelta(days=30)
 
-    query = Document2.objects.get(id=query_id)
-    stats.append(('Query', '%s - %s' % (query_id, query.name)))
-    stats.append(('Executions', query.dependents.filter(is_history=True, type__startswith='query-').count()))
+    query = Document2.objects.get(id=query_id) if query is None else query
+    stats.append({
+      'name': 'query',
+      'value': '%s - %s' % (query_id, query.name),
+      'description': _('Query info')
+    })
+    executions = query.dependents.filter(is_history=True, type__startswith='query-')
+    stats.append({
+      'name': 'execution_count',
+      'value': executions.count(),
+      'description': _('How many times executed')
+    })
+    stats.append({
+      'name': 'execution_count_shared',
+      'value': executions.exclude(owner=query.owner).count(),
+      'description': _('Executions by others')
+    })
+    last_month_daily = executions.filter(last_modified__gte=one_month).annotate(day=Trunc('last_modified', 'day')).values('day').annotate(c=Count('day')).values('day', 'c').order_by('day')
+    stats.append({
+      'name': 'executions_30_days_histogram',
+      'value': last_month_daily,
+      'description': _('Daily executions 30 days')
+    })
+    # Could count number of "forks" (but would need to start tracking parent of Saved As query cf. saveAsNotebook)
 
     return stats

+ 19 - 6
desktop/libs/notebook/src/notebook/tests.py

@@ -18,6 +18,7 @@
 
 import json
 
+from collections import OrderedDict
 from nose.tools import assert_equal, assert_true, assert_false
 
 from django.contrib.auth.models import User
@@ -37,12 +38,7 @@ from notebook.api import _historify
 from notebook.connectors.base import Notebook, QueryError, Api
 from notebook.decorators import api_error_handler
 from notebook.conf import get_ordered_interpreters, INTERPRETERS_SHOWN_ON_WHEEL, INTERPRETERS
-
-
-try:
-  from collections import OrderedDict
-except ImportError:
-  from ordereddict import OrderedDict # Python 2.6
+from notebook.models import Analytics
 
 
 class TestNotebookApi(object):
@@ -478,3 +474,20 @@ def test_get_interpreters_to_show():
     appmanager.DESKTOP_MODULES = []
     appmanager.DESKTOP_APPS = None
     appmanager.load_apps(APP_BLACKLIST.get())
+
+
+class TestAnalytics():
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
+    self.user = User.objects.get(username="test")
+
+  def test_basic_stats(self):
+    try:
+      doc, created = Document2.objects.get_or_create(name='test_query_stats', type='query-hive', owner=self.user, data={})
+
+      Analytics.admin_stats()
+      Anaytics.user_stats(user=user)
+      Anaytics.query_stats(query=doc)
+    finally:
+      doc.delete()