Przeglądaj źródła

[notebook] The default list of interpreter is not cached properly

Here we get the proper list of default interpreters.
Before this list depended on the interpreters of the first user landing
on the server.

And now the default list is then filtered properly with the permissions
of the loged in user.

Note that `_default_interpreters(user)` was auto caching the values
which made successsive call lose the extra interpreters.

This was also a slow operation (happening quite often).

All this logic is gone and simpler when using the new connectors.
Romain 5 lat temu
rodzic
commit
a0f186b2bc

+ 32 - 14
desktop/libs/notebook/src/notebook/api_tests.py

@@ -21,7 +21,6 @@ import json
 import sys
 
 from collections import OrderedDict
-from datetime import datetime
 from nose.plugins.attrib import attr
 from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal, assert_true, assert_false
@@ -39,6 +38,7 @@ from desktop.models import Directory, Document, Document2
 from hadoop import cluster as originalCluster
 from useradmin.models import User
 
+import notebook.conf
 import notebook.connectors.hiveserver2
 
 from notebook.api import _historify
@@ -677,28 +677,27 @@ def test_get_interpreters_to_show():
     ))
 
   try:
-    resets = [INTERPRETERS.set_for_testing(default_interpreters), APP_BLACKLIST.set_for_testing('')]
+    resets = [
+      INTERPRETERS.set_for_testing(default_interpreters),
+      APP_BLACKLIST.set_for_testing(''),
+      ENABLE_CONNECTORS.set_for_testing(False)
+    ]
     appmanager.DESKTOP_MODULES = []
     appmanager.DESKTOP_APPS = None
     appmanager.load_apps(APP_BLACKLIST.get())
+    notebook.conf.INTERPRETERS_CACHE = None
 
-    interpreters_shown_on_wheel_unset = get_ordered_interpreters()
     assert_equal(
-      list(default_interpreters.values()),
-      interpreters_shown_on_wheel_unset,
-      'get_interpreters_to_show should return the same as get_interpreters when interpreters_shown_on_wheel '
-      'is unset. expected: %s, actual: %s' % (
-          list(default_interpreters.values()), interpreters_shown_on_wheel_unset
-      )
+      list(default_interpreters.values()), get_ordered_interpreters(),
+      'get_interpreters_to_show should return the same as get_interpreters when interpreters_shown_on_wheel is unset'
     )
 
+
     resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig'))
+
     assert_equal(
-      list(expected_interpreters.values()),
-      get_ordered_interpreters(),
-      'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s' % (
-          list(expected_interpreters.values()), get_ordered_interpreters()
-      )
+      list(expected_interpreters.values()), get_ordered_interpreters(),
+      'get_interpreters_to_show did not return interpreters in the correct order expected'
     )
   finally:
     for reset in resets:
@@ -706,6 +705,7 @@ def test_get_interpreters_to_show():
     appmanager.DESKTOP_MODULES = []
     appmanager.DESKTOP_APPS = None
     appmanager.load_apps(APP_BLACKLIST.get())
+    notebook.conf.INTERPRETERS_CACHE = None
 
 
 def test_get_ordered_interpreters():
@@ -743,6 +743,7 @@ def test_get_ordered_interpreters():
         with patch('notebook.conf.has_connectors') as has_connectors:
           get_apps_dict.return_value = {'hive': {}}
           has_connectors.return_value = False
+          notebook.conf.INTERPRETERS_CACHE = None
 
           is_cm_managed.return_value = False
 
@@ -758,18 +759,30 @@ def test_get_ordered_interpreters():
             [interpreter['dialect'] for interpreter in get_ordered_interpreters()],
             ['phoenix']
           )
+          assert_equal(  # Check twice because of cache
+            [interpreter['dialect'] for interpreter in get_ordered_interpreters()],
+            ['phoenix']
+          )
 
           is_cm_managed.return_value = True
+          notebook.conf.INTERPRETERS_CACHE = None
 
           # CM --> Append []
           INTERPRETERS.set_for_testing(
             OrderedDict(()
             )
           )
+
           assert_equal(
             [interpreter['dialect'] for interpreter in get_ordered_interpreters()],
             ['hive']
           )
+          assert_equal(  # Check twice
+            [interpreter['dialect'] for interpreter in get_ordered_interpreters()],
+            ['hive']
+          )
+
+          notebook.conf.INTERPRETERS_CACHE = None
 
           # CM --> Append [Phoenix]
           INTERPRETERS.set_for_testing(
@@ -783,12 +796,17 @@ def test_get_ordered_interpreters():
             [interpreter['dialect'] for interpreter in get_ordered_interpreters()],
             ['hive', 'phoenix']
           )
+          assert_equal(  # Check twice
+            [interpreter['dialect'] for interpreter in get_ordered_interpreters()],
+            ['hive', 'phoenix']
+          )
   finally:
     for reset in resets:
       reset()
     appmanager.DESKTOP_MODULES = []
     appmanager.DESKTOP_APPS = None
     appmanager.load_apps(APP_BLACKLIST.get())
+    notebook.conf.INTERPRETERS_CACHE = None
 
 
 class TestQueriesMetrics(object):

+ 21 - 14
desktop/libs/notebook/src/notebook/conf.py

@@ -31,6 +31,9 @@ from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, co
 
 LOG = logging.getLogger(__name__)
 
+# Not used when connector are on
+INTERPRETERS_CACHE = None
+
 
 SHOW_NOTEBOOKS = Config(
     key="show_notebooks",
@@ -70,6 +73,8 @@ def _connector_to_iterpreter(connector):
 
 
 def get_ordered_interpreters(user=None):
+  global INTERPRETERS_CACHE
+
   if has_connectors():
     from desktop.lib.connectors.api import _get_installed_connectors
     interpreters = [
@@ -77,21 +82,22 @@ def get_ordered_interpreters(user=None):
       for connector in _get_installed_connectors(categories=['editor', 'catalogs'], user=user)
     ]
   else:
-    if is_cm_managed() and INTERPRETERS.get():
-      extra_interpreters = INTERPRETERS.get()
-      _default_interpreters(user)
-    else:
-      extra_interpreters = {}
+    if INTERPRETERS_CACHE is None:
+      if is_cm_managed():
+        extra_interpreters = INTERPRETERS.get()  # Combine the other apps interpreters
+        _default_interpreters(user)
+      else:
+        extra_interpreters = {}
 
-    if not INTERPRETERS.get():
-      _default_interpreters(user)
+      if not INTERPRETERS.get():
+        _default_interpreters(user)
 
-    interpreters = INTERPRETERS.get()
-    interpreters.update(extra_interpreters)
+      INTERPRETERS_CACHE = INTERPRETERS.get()
+      INTERPRETERS_CACHE.update(extra_interpreters)
 
     user_apps = appmanager.get_apps_dict(user)
     user_interpreters = []
-    for interpreter in interpreters:
+    for interpreter in INTERPRETERS_CACHE:
       if check_has_missing_permission(user, interpreter, user_apps=user_apps):
         pass  # Not allowed
       else:
@@ -106,11 +112,12 @@ def get_ordered_interpreters(user=None):
     reordered_interpreters = interpreters_shown_on_wheel + [i for i in user_interpreters if i not in interpreters_shown_on_wheel]
 
     interpreters = [{
-        'name': interpreters[i].NAME.get(),
+        'name': INTERPRETERS_CACHE[i].NAME.get(),
         'type': i,
-        'interface': interpreters[i].INTERFACE.get(),
-        'options': interpreters[i].OPTIONS.get()
-      } for i in reordered_interpreters
+        'interface': INTERPRETERS_CACHE[i].INTERFACE.get(),
+        'options': INTERPRETERS_CACHE[i].OPTIONS.get()
+      }
+      for i in reordered_interpreters
     ]
 
   return [{