Forráskód Böngészése

HUE-5478 [notebook] Allow users to specify interpreters on Notebook selection wheel (#461)

https://github.com/cloudera/hue/pull/461
z-york 9 éve
szülő
commit
d744c47429

+ 5 - 1
desktop/conf.dist/hue.ini

@@ -634,7 +634,7 @@
   ## Client Secret for Authorized GitHub Application
   # github_client_secret=
 
-  # One entry for each type of snippet. The first 5 will appear in the wheel.
+  # One entry for each type of snippet.
   [[interpreters]]
     # Define the name and how to connect and execute the language.
 
@@ -744,6 +744,10 @@
   ## Classpath to be appended to the default DBProxy server classpath.
   # dbproxy_extra_classpath=
 
+  ## Comma separated list of interpreters that should be shown on the wheel. This list takes precedence over the
+  ## order in which the interpreter entries appear. Only the first 5 interpreters will appear on the wheel.
+  # interpreters_shown_on_wheel=
+
 
 ###########################################################################
 # Settings to configure your Hadoop cluster.

+ 5 - 1
desktop/conf/pseudo-distributed.ini.tmpl

@@ -638,7 +638,7 @@
   ## Client Secret for Authorized GitHub Application
   # github_client_secret=
 
-  # One entry for each type of snippet. The first 5 will appear in the wheel.
+  # One entry for each type of snippet.
   [[interpreters]]
     # Define the name and how to connect and execute the language.
 
@@ -748,6 +748,10 @@
   ## Classpath to be appended to the default DBProxy server classpath.
   # dbproxy_extra_classpath=
 
+  ## Comma separated list of interpreters that should be shown on the wheel. This list takes precedence over the
+  ## order in which the interpreter entries appear. Only the first 5 interpreters will appear on the wheel.
+  # interpreters_shown_on_wheel=
+
 
 ###########################################################################
 # Settings to configure your Hadoop cluster.

+ 37 - 16
desktop/libs/notebook/src/notebook/conf.py

@@ -24,23 +24,9 @@ from django.utils.translation import ugettext_lazy as _t
 
 from desktop import appmanager
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection,\
-  coerce_json_dict, coerce_string, coerce_bool
+  coerce_json_dict, coerce_string, coerce_bool, coerce_csv
 
 
-def get_interpreters(user=None):
-  if not INTERPRETERS.get():
-    _default_interpreters()
-
-  interpreters = INTERPRETERS.get()
-
-  return [{
-      "name": interpreters[i].NAME.get(),
-      "type": i,
-      "interface": interpreters[i].INTERFACE.get(),
-      "options": interpreters[i].OPTIONS.get()}
-      for i in interpreters
-  ]
-
 def is_oozie_enabled():
   """Oozie needs to be available as it is the backend."""
   return len([app for app in appmanager.DESKTOP_MODULES if app.name == 'oozie']) > 0
@@ -53,9 +39,35 @@ SHOW_NOTEBOOKS = Config(
     default=True
 )
 
+def _remove_duplications(a_list):
+  return list(OrderedDict.fromkeys(a_list))
+
+def get_ordered_interpreters(user=None):
+  if not INTERPRETERS.get():
+    _default_interpreters()
+
+  interpreters = INTERPRETERS.get()
+  interpreters_shown_on_wheel = _remove_duplications(INTERPRETERS_SHOWN_ON_WHEEL.get())
+
+  unknown_interpreters = set(interpreters_shown_on_wheel) - set(interpreters)
+  if unknown_interpreters:
+      raise ValueError("Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s"
+                       % unknown_interpreters)
+
+  reordered_interpreters = interpreters_shown_on_wheel + \
+                           [i for i in interpreters if i not in interpreters_shown_on_wheel]
+
+  return [{
+      "name": interpreters[i].NAME.get(),
+      "type": i,
+      "interface": interpreters[i].INTERFACE.get(),
+      "options": interpreters[i].OPTIONS.get()}
+      for i in reordered_interpreters
+  ]
+
 INTERPRETERS = UnspecifiedConfigSection(
   "interpreters",
-  help="One entry for each type of snippet. The first 5 will appear in the wheel.",
+  help="One entry for each type of snippet.",
   each=ConfigSection(
     help=_t("Define the name and how to connect and execute the language."),
     members=dict(
@@ -81,6 +93,15 @@ INTERPRETERS = UnspecifiedConfigSection(
   )
 )
 
+INTERPRETERS_SHOWN_ON_WHEEL = Config(
+  key="interpreters_shown_on_wheel",
+  help=_t("Comma separated list of interpreters that should be shown on the wheel. "
+          "This list takes precedence over the order in which the interpreter entries appear. "
+          "Only the first 5 interpreters will appear on the wheel."),
+  type=coerce_csv,
+  default=[]
+)
+
 ENABLE_DBPROXY_SERVER = Config(
   key="enable_dbproxy_server",
   help=_t("Main flag to override the automatic starting of the DBProxy server."),

+ 2 - 2
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -24,7 +24,7 @@ from django.utils.translation import ugettext as _
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_unicode
 
-from notebook.conf import get_interpreters
+from notebook.conf import get_ordered_interpreters
 
 
 LOG = logging.getLogger(__name__)
@@ -199,7 +199,7 @@ def get_api(request, snippet):
   if snippet.get('wasBatchExecuted'):
     return OozieApi(user=request.user, request=request)
 
-  interpreter = [interpreter for interpreter in get_interpreters(request.user) if interpreter['type'] == snippet['type']]
+  interpreter = [interpreter for interpreter in get_ordered_interpreters(request.user) if interpreter['type'] == snippet['type']]
   if not interpreter:
     raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet)
   interpreter = interpreter[0]

+ 2 - 2
desktop/libs/notebook/src/notebook/monkey_patches.py

@@ -17,7 +17,7 @@
 
 # Start DBProxy server if we have some JDBC snippets
 
-from notebook.conf import get_interpreters, ENABLE_DBPROXY_SERVER
+from notebook.conf import get_ordered_interpreters, ENABLE_DBPROXY_SERVER
 
 
 def _start_livy_server():
@@ -41,5 +41,5 @@ def _start_livy_server():
   atexit.register(cleanup)
 
 
-if ENABLE_DBPROXY_SERVER.get() and [interpreter for interpreter in get_interpreters() if interpreter['interface'] == 'jdbc']:
+if ENABLE_DBPROXY_SERVER.get() and [interpreter for interpreter in get_ordered_interpreters() if interpreter['interface'] == 'jdbc']:
   _start_livy_server()

+ 55 - 0
desktop/libs/notebook/src/notebook/tests.py

@@ -33,6 +33,12 @@ import notebook.connectors.hiveserver2
 from notebook.api import _historify
 from notebook.connectors.base import Notebook, QueryError, Api
 from notebook.decorators import api_error_handler
+from notebook.conf import get_ordered_interpreters, INTERPRETERS_SHOWN_ON_WHEEL, INTERPRETERS
+
+try:
+  from collections import OrderedDict
+except ImportError:
+  from ordereddict import OrderedDict # Python 2.6
 
 
 class TestNotebookApi(object):
@@ -358,3 +364,52 @@ class TestNotebookApiMocked(object):
     data = json.loads(response.content)
     assert_equal(0, data['status'], data)
     assert_equal('/user/hue/path.csv', data['watch_url']['destination'], data)
+
+
+def test_get_interpreters_to_show():
+  default_interpreters = OrderedDict((
+      ('hive', {
+          'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'options': {}
+      }),
+      ('spark', {
+          'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'options': {}
+      }),
+      ('pig', {
+          'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'options': {}
+      }),
+      ('java', {
+          'name': 'Java', 'interface': 'oozie', 'type': 'java', 'options': {}
+      })
+    ))
+
+  expected_interpreters = OrderedDict((
+      ('java', {
+        'name': 'Java', 'interface': 'oozie', 'type': 'java', 'options': {}
+      }),
+      ('pig', {
+        'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'options': {}
+      }),
+      ('hive', {
+          'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'options': {}
+      }),
+      ('spark', {
+          'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'options': {}
+      })
+    ))
+
+  try:
+    resets = [INTERPRETERS.set_for_testing(default_interpreters)]
+
+    interpreters_shown_on_wheel_unset = get_ordered_interpreters()
+    assert_equal(default_interpreters.values(), interpreters_shown_on_wheel_unset,
+                 'get_interpreters_to_show should return the same as get_interpreters when '
+                 'interpreters_shown_on_wheel is unset. expected: %s, actual: %s'
+                 % (default_interpreters.values(), interpreters_shown_on_wheel_unset))
+
+    resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig'))
+    assert_equal(expected_interpreters.values(), get_ordered_interpreters(),
+                 'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s'
+                 % (expected_interpreters.values(), get_ordered_interpreters()))
+  finally:
+    for reset in resets:
+      reset()

+ 2 - 2
desktop/libs/notebook/src/notebook/views.py

@@ -31,7 +31,7 @@ from desktop.models import Document2, Document
 
 from metadata.conf import has_optimizer, has_navigator
 
-from notebook.conf import get_interpreters
+from notebook.conf import get_ordered_interpreters
 from notebook.connectors.base import Notebook, get_api
 from notebook.connectors.spark_shell import SparkApi
 from notebook.decorators import check_document_access_permission, check_document_modify_permission
@@ -81,7 +81,7 @@ def notebook(request, is_embeddable=False):
       'editor_id': notebook_id or None,
       'notebooks_json': '{}',
       'options_json': json.dumps({
-          'languages': get_interpreters(request.user),
+          'languages': get_ordered_interpreters(request.user),
           'session_properties': SparkApi.get_properties(),
           'is_optimizer_enabled': has_optimizer(),
           'is_navigator_enabled': has_navigator(request.user),