Ver Fonte

HUE-9369 [editor] Configuration to auto install sample queries and tables

At each startup (note: assumes connectivity with DB works) Hue will check and install those.

Bit simpler, by default assuming the dialect list from the installed connectors.
Assumes we improve examples format with a name id.

Automatically filter out examples per dialect (already done).
Take the opportunity to add required tables into the query sample data.

e.g.

[notebook]
  [[examples]]
  auto_load=true
  queries='Sample: Top salary', 'Queries 2'
  tables=

  [[interpreters]]
  ....
Romain há 5 anos atrás
pai
commit
71e1a9542e

+ 21 - 17
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples.py

@@ -62,6 +62,8 @@ class Command(BaseCommand):
       interpreter = options.get('interpreter')  # Only when connectors are enabled. Later will deprecate `dialect`.
       user = options['user']
       request = options['request']
+      self.queries = options.get('queries')
+      self.tables = options.get('tables')
 
     tables = 'tables_standard.json' if dialect not in ('hive', 'impala') else (
         'tables_transactional.json' if has_concurrency_support() else 'tables.json'
@@ -104,15 +106,16 @@ class Command(BaseCommand):
       raise InstallException(_('No %s tables are available as samples') % dialect)
 
     for table_dict in table_list:
-      full_name = '%s.%s' % (db_name, table_dict['table_name'])
-      try:
-        table = SampleTable(table_dict, dialect, db_name, interpreter=interpreter, request=request)
-        if table.install(django_user):
-          self.successes.append(_('Table %s installed.') % full_name)
-      except Exception as ex:
-        msg = str(ex)
-        LOG.error(msg)
-        self.errors.append(_('Could not install table %s: %s') % (full_name, msg))
+      if self.tables is None or table_dict['table_name'] in self.tables:
+        full_name = '%s.%s' % (db_name, table_dict['table_name'])
+        try:
+          table = SampleTable(table_dict, dialect, db_name, interpreter=interpreter, request=request)
+          if table.install(django_user):
+            self.successes.append(_('Table %s installed.') % full_name)
+        except Exception as ex:
+          msg = str(ex)
+          LOG.error(msg)
+          self.errors.append(_('Could not install table %s: %s') % (full_name, msg))
 
 
   def install_queries(self, django_user, dialect, interpreter=None):
@@ -130,14 +133,15 @@ class Command(BaseCommand):
       raise InstallException(_('No %s queries are available as samples') % dialect)
 
     for design_dict in design_list:
-      design = SampleQuery(design_dict)
-      try:
-        design.install(django_user, interpreter=interpreter)
-        self.successes.append(_('Query %s %s installed.') % (design_dict['name'], dialect))
-      except Exception as ex:
-        msg = str(ex)
-        LOG.error(msg)
-        self.errors.append(_('Could not install %s query: %s') % (dialect, msg))
+      if self.queries is None or design_dict['name'] in self.queries:
+        design = SampleQuery(design_dict)
+        try:
+          design.install(django_user, interpreter=interpreter)
+          self.successes.append(_('Query %s %s installed.') % (design_dict['name'], dialect))
+        except Exception as ex:
+          msg = str(ex)
+          LOG.error(msg)
+          self.errors.append(_('Could not install %s query: %s') % (dialect, msg))
 
 
 class SampleTable(object):

+ 9 - 0
desktop/conf.dist/hue.ini

@@ -1041,6 +1041,15 @@
     #   ## If 'user' and 'password' are omitted, they will be prompted in the UI.
     #   options='{"url": "jdbc:vertica://localhost:5434", "driver": "com.vertica.jdbc.Driver"}'
 
+    ## Define which query and table examples can be automatically setup for the available dialects.
+    # [[examples]]
+    ## If installing the examples automatically at startup.
+    # auto_load=false
+    ## Names of the saved queries to install. All if empty.
+    # queries=
+    ## Names of the tables to install. All if empty.
+    # tables=
+
 
 ###########################################################################
 # Settings to configure your Analytics Dashboards

+ 9 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1025,6 +1025,15 @@
     #   ## If 'user' and 'password' are omitted, they will be prompted in the UI.
     #   options='{"url": "jdbc:vertica://localhost:5434", "driver": "com.vertica.jdbc.Driver"}'
 
+    ## Define which query and table examples can be automatically setup for the available dialects.
+    # [[examples]]
+    ## If installing the examples automatically at startup.
+    # auto_load=false
+    ## Names of the saved queries to install. All if empty.
+    # queries=
+    ## Names of the tables to install. All if empty.
+    # tables=
+
 
 ###########################################################################
 # Settings to configure your Analytics Dashboards

+ 3 - 1
desktop/core/src/desktop/models.py

@@ -1749,7 +1749,9 @@ class ClusterConfig(object):
     default_interpreter = default_app.get('interpreters')
 
     try:
-      user_default_app = json.loads(UserPreferences.objects.get(user=self.user, key='default_app').value)
+      user_default_app = json.loads(
+        UserPreferences.objects.get(user=self.user, key='default_app').value
+      )
       if apps.get(user_default_app['app']):
         default_interpreter = []
         default_app = apps[user_default_app['app']]

+ 2 - 0
desktop/libs/notebook/src/notebook/__init__.py

@@ -13,3 +13,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+default_app_config = 'notebook.apps.NotebookConfig'

+ 1 - 2
desktop/libs/notebook/src/notebook/api.py

@@ -127,15 +127,14 @@ def _execute_notebook(request, notebook, snippet):
   response = {'status': -1}
   result = None
   history = None
+  active_executable = None
 
   historify = (notebook['type'] != 'notebook' or snippet.get('wasBatchExecuted')) and not notebook.get('skipHistorify')
 
   try:
     try:
       sessions = notebook.get('sessions') and notebook['sessions'] # Session reference for snippet execution without persisting it
-
       active_executable = json.loads(request.POST.get('executable', '{}')) # Editor v2
-
       # TODO: Use statement, database etc. from active_executable
 
       if historify:

+ 33 - 0
desktop/libs/notebook/src/notebook/apps.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from django.apps import AppConfig
+
+
+LOG = logging.getLogger(__name__)
+
+
+class NotebookConfig(AppConfig):
+  name = 'notebook'
+  verbose_name = "Notebook SQL Assistant"
+
+  def ready(self):
+    from notebook.models import install_custom_examples
+
+    install_custom_examples()

+ 25 - 0
desktop/libs/notebook/src/notebook/conf.py

@@ -237,6 +237,31 @@ ENABLE_QUERY_ANALYSIS = Config(
 )
 
 
+EXAMPLES = ConfigSection(
+  key='examples',
+  help=_t('Define which query and table examples can be automatically setup for the available dialects.'),
+  members=dict(
+    AUTO_LOAD=Config(
+      'auto_load',
+      help=_t('If installing the examples automatically at startup.'),
+      type=coerce_bool,
+      default=False
+    ),
+    QUERIES=Config(
+      'queries',
+      help='Names of the saved queries to install. All if empty.',
+      type=coerce_csv,
+      default=[]
+    ),
+    TABLES=Config(
+      key='tables',
+      help=_t('Names of the tables to install. All if empty.'),
+      type=coerce_csv,
+      default=[]
+    )
+  )
+)
+
 def _default_interpreters(user):
   interpreters = []
   apps = appmanager.get_apps_dict(user)

+ 50 - 0
desktop/libs/notebook/src/notebook/models.py

@@ -42,6 +42,7 @@ from desktop.lib.paths import SAFE_CHARACTERS_URI
 from desktop.models import Document2
 from useradmin.models import User
 
+from notebook.conf import EXAMPLES, get_ordered_interpreters
 from notebook.connectors.base import Notebook, get_api as _get_api, get_interpreter
 
 if sys.version_info[0] > 2:
@@ -633,3 +634,52 @@ class MockRequest():
   def __init__(self, user, ):
     self.user = user
     self.POST = {}
+
+
+def install_custom_examples():
+  if EXAMPLES.AUTO_LOAD.get():
+    from desktop.auth.backend import rewrite_user
+    from beeswax.management.commands import beeswax_install_examples
+    from useradmin.models import get_default_user_group, install_sample_user, User
+
+    user = rewrite_user(
+      install_sample_user()
+    )
+
+    dialects = [
+      interpreter['dialect']
+      for interpreter in get_ordered_interpreters(user)
+      if interpreter['dialect'] in ('hive', 'impala')  # Only for hive/impala currently, would also need to port to Notebook install examples.
+    ]
+
+    queries = EXAMPLES.QUERIES.get()
+    tables = EXAMPLES.TABLES.get()  # No-op. Only for the saved query samples, not the tables currently.
+
+    LOG.info('Installing custom examples queries: %(queries)s, tables: %(tables)s for dialects %(dialects)s belonging to user %(user)s' % {
+      'queries': queries,
+      'tables': tables,
+      'dialects': dialects,
+      'user': user
+    })
+
+    result = []
+
+    for dialect in dialects:
+      interpreter = {'type': dialect, 'dialect': dialect}
+
+      successes, errors = beeswax_install_examples.Command().handle(
+          dialect=dialect,
+          user=user,
+          interpreter=interpreter,
+          queries=queries,
+          tables=tables,
+          request=None
+      )
+      LOG.info('Dialect %(dialect)s: %(successes)s, %(errors)s,' % {
+        'dialect': dialect,
+        'successes': successes,
+        'errors': errors,
+      })
+      result.append((successes, errors))
+
+    return result

+ 94 - 0
desktop/libs/notebook/src/notebook/models_tests.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import json
+import sys
+
+from django.urls import reverse
+from nose.tools import assert_equal, assert_not_equal, assert_true, assert_false
+
+from desktop.lib.django_test_util import make_logged_in_client
+from desktop.lib.connectors.models import Connector
+from useradmin.models import User
+
+from notebook.conf import EXAMPLES
+from notebook.models import install_custom_examples
+
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock, MagicMock
+else:
+  from mock import patch, Mock, MagicMock
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TestInstallCustomExamples():
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=True, is_admin=True)
+    self.user = User.objects.get(username="test")
+
+
+  def test_install_only_hive_queries(self):
+    f = EXAMPLES.AUTO_LOAD.set_for_testing(True)
+
+    try:
+      with patch('notebook.models.get_ordered_interpreters') as get_ordered_interpreters:
+        with patch('notebook.models.EXAMPLES.QUERIES.get') as QUERIES:
+
+          QUERIES.return_value = ['Sample: Top salary', 'Queries Does not Exist']
+
+          get_ordered_interpreters.return_value = [
+            {
+              'name': 'Hive',
+              'type': 11,
+              'dialect': 'hive',
+              'interface': 'hiveserver2',
+            },
+            {
+              'name': 'MySql',
+              'type': 10,
+              'dialect': 'mysql',
+              'interface': 'sqlalchemy',
+            }
+          ]
+
+          result = install_custom_examples()
+
+          assert_equal(1, len(result))
+          successes, errors = result[0]
+
+          assert_equal([], errors)
+          assert_equal(
+              ['Query Sample: Top salary hive installed.'],
+              successes,
+          )
+    finally:
+      f()
+
+
+  def test_install_auto_load_disabled(self):
+    f = EXAMPLES.AUTO_LOAD.set_for_testing(False)
+    try:
+      result = install_custom_examples()
+
+      assert_true(result is None, result)
+    finally:
+      f()