浏览代码

HUE-8758 [connector] Install examples for the particular connector

Romain 5 年之前
父节点
当前提交
94dda0f280

+ 3 - 3
apps/about/src/about/templates/admin_wizard.mako

@@ -316,11 +316,11 @@ ${ layout.menubar(section='quick_start') }
     self.connectors = ko.observableArray();
     self.isInstallingSample = ko.observable(false);
 
-    self.installConnectorDataExample = function(data, event) {
+    self.installConnectorDataExample = function(connector, event) {
       self.isInstallingSample(true);
 
       $.post("${ url('notebook:install_examples') }", {
-          dialect: data.dialect
+          connector: connector.type
         }, function(data) {
         if (data.status == 0) {
           $(document).trigger('info','${ _("Examples refreshed") }');
@@ -427,7 +427,7 @@ ${ layout.menubar(section='quick_start') }
 
     function showStep(step) {
       if (window.location.hash === '#step1' || window.location.hash === '') {
-        checkConfig()
+        checkConfig();
       }
 
       currentStep = step;

+ 17 - 16
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples.py

@@ -32,10 +32,10 @@ from hadoop import cluster
 from notebook.models import import_saved_beeswax_query
 from useradmin.models import get_default_user_group, install_sample_user, User
 
-import beeswax.conf
-from beeswax.models import SavedQuery, HQL, IMPALA
 from beeswax.design import hql_query
+from beeswax.conf import LOCAL_EXAMPLES_DATA_DIR
 from beeswax.hive_site import has_concurrency_support
+from beeswax.models import SavedQuery, HQL, IMPALA
 from beeswax.server import dbms
 from beeswax.server.dbms import get_query_server_config, QueryServerException
 
@@ -57,6 +57,7 @@ class Command(BaseCommand):
       db_name = args[1] if len(args) > 1 else 'default'
       user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
     else:
+      interpreter = options.get('interpreter')
       app_name = options['app_name']
       db_name = options.get('db_name', 'default')
       user = options['user']
@@ -68,8 +69,8 @@ class Command(BaseCommand):
     # Documents will belong to this user but we run the install as the current user
     try:
       sample_user = install_sample_user(user)
-      self._install_queries(sample_user, app_name)
-      self._install_tables(user, app_name, db_name, tables)
+      self._install_queries(sample_user, app_name, interpreter=interpreter)
+      self._install_tables(user, app_name, db_name, tables, interpreter=interpreter)
     except Exception as ex:
       exception = ex
 
@@ -88,21 +89,21 @@ class Command(BaseCommand):
       else:
         raise exception
 
-  def _install_tables(self, django_user, app_name, db_name, tables):
-    data_dir = beeswax.conf.LOCAL_EXAMPLES_DATA_DIR.get()
+  def _install_tables(self, django_user, app_name, db_name, tables, interpreter=None):
+    data_dir = LOCAL_EXAMPLES_DATA_DIR.get()
     table_file = open(os.path.join(data_dir, tables))
     table_list = json.load(table_file)
     table_file.close()
 
     for table_dict in table_list:
-      table = SampleTable(table_dict, app_name, db_name)
+      table = SampleTable(table_dict, app_name, db_name, interpreter=interpreter)
       try:
         table.install(django_user)
       except Exception as ex:
         raise InstallException(_('Could not install table: %s') % ex)
 
-  def _install_queries(self, django_user, app_name):
-    design_file = open(os.path.join(beeswax.conf.LOCAL_EXAMPLES_DATA_DIR.get(), 'designs.json'))
+  def _install_queries(self, django_user, app_name, interpreter=None):
+    design_file = open(os.path.join(LOCAL_EXAMPLES_DATA_DIR.get(), 'designs.json'))
     design_list = json.load(design_file)
     design_file.close()
 
@@ -113,16 +114,16 @@ class Command(BaseCommand):
     for design_dict in design_list:
       design = SampleQuery(design_dict)
       try:
-        design.install(django_user)
+        design.install(django_user, interpreter=interpreter)
       except Exception as ex:
-        raise InstallException(_('Could not install query: %s') % ex)
+        raise InstallException(_('Could not install %s query: %s') % (app_name, ex))
 
 
 class SampleTable(object):
   """
   Represents a table loaded from the tables.json file
   """
-  def __init__(self, data_dict, app_name, db_name='default'):
+  def __init__(self, data_dict, app_name, db_name='default', interpreter=None):
     self.name = data_dict['table_name']
     if 'partition_files' in data_dict:
       self.partition_files = data_dict['partition_files']
@@ -130,14 +131,14 @@ class SampleTable(object):
       self.partition_files = None
       self.filename = data_dict['data_file']
     self.hql = data_dict['create_hql']
-    self.query_server = get_query_server_config(app_name)
+    self.query_server = get_query_server_config(app_name, connector=interpreter)
     self.app_name = app_name
     self.db_name = db_name
     self.columns = data_dict.get('columns')
     self.is_transactional = data_dict.get('transactional')
 
     # Sanity check
-    self._data_dir = beeswax.conf.LOCAL_EXAMPLES_DATA_DIR.get()
+    self._data_dir = LOCAL_EXAMPLES_DATA_DIR.get()
     if self.partition_files:
       for partition_spec, filename in list(self.partition_files.items()):
         filepath = os.path.join(self._data_dir, filename)
@@ -341,7 +342,7 @@ class SampleQuery(object):
     self.data = data_dict['data']
 
 
-  def install(self, django_user):
+  def install(self, django_user, interpreter=None):
     """
     Install queries. Raise InstallException on failure.
     """
@@ -376,7 +377,7 @@ class SampleQuery(object):
           doc2.save()
       except Document2.DoesNotExist:
         # Create document from saved query
-        notebook = import_saved_beeswax_query(query)
+        notebook = import_saved_beeswax_query(query, interpreter=interpreter)
         data = notebook.get_data()
         data['isSaved'] = True
         uuid = data.get('uuid')

+ 1 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -207,6 +207,7 @@ def get_query_server_config(name='beeswax', connector=None):
 
 
 def get_query_server_config_via_connector(connector):
+  # TODO: connector is actually a notebook interpreter
   connector_name = full_connector_name = connector['type']
   compute_name = None
   if connector.get('compute'):

+ 3 - 2
apps/beeswax/src/beeswax/views.py

@@ -49,9 +49,9 @@ from useradmin.models import User
 
 import beeswax.forms
 import beeswax.design
-import beeswax.management.commands.beeswax_install_examples
 
 from beeswax import common, data_export, models
+from beeswax.management.commands import beeswax_install_examples
 from beeswax.models import QueryHistory, SavedQuery, Session
 from beeswax.server import dbms
 from beeswax.server.dbms import expand_exception, get_query_server_config, QueryServerException
@@ -614,7 +614,8 @@ def install_examples(request):
     try:
       app_name = get_app_name(request)
       db_name = request.POST.get('db_name', 'default')
-      beeswax.management.commands.beeswax_install_examples.Command().handle(app_name=app_name, db_name=db_name, user=request.user)
+      connector_id = options.get('connector_id')
+      beeswax_install_examples.Command().handle(app_name=app_name, db_name=db_name, user=request.user)
       response['status'] = 0
     except Exception as err:
       LOG.exception(err)

+ 10 - 7
apps/useradmin/src/useradmin/models.py

@@ -326,7 +326,7 @@ def install_sample_user(django_user=None):
     django_username_short = django_username
 
   try:
-    if User.objects.filter(id=SAMPLE_USER_ID).exists():
+    if User.objects.filter(id=SAMPLE_USER_ID).exists() and not ENABLE_ORGANIZATIONS.get():
       user = User.objects.get(id=SAMPLE_USER_ID)
       LOG.info('Sample user found with username "%s" and User ID: %s' % (user.username, user.id))
     elif User.objects.filter(**lookup).exists():
@@ -336,12 +336,13 @@ def install_sample_user(django_user=None):
       user_attributes = lookup.copy()
       if ENABLE_ORGANIZATIONS.get():
         user_attributes['organization'] = get_organization(email=django_username)
+      else:
+        user_attributes['id'] = SAMPLE_USER_ID
+
       user_attributes.update({
         'password': '!',
         'is_active': False,
         'is_superuser': False,
-        'id': SAMPLE_USER_ID,
-        'pk': SAMPLE_USER_ID
       })
       user, created = User.objects.get_or_create(**user_attributes)
 
@@ -354,19 +355,21 @@ def install_sample_user(django_user=None):
         user = User.objects.get(id=SAMPLE_USER_ID)
         user.username = django_username
         user.save()
-  except Exception as ex:
+  except:
     LOG.exception('Failed to get or create sample user')
 
   # If sample user doesn't belong to default group, add to default group
-  default_group = get_default_user_group()
+  default_group = get_default_user_group(user=user)
   if user is not None and default_group is not None and default_group not in user.groups.all():
     user.groups.add(default_group)
     user.save()
 
-  fs = cluster.get_hdfs()
   # If home directory doesn't exist for sample user, create it
+  fs = cluster.get_hdfs()
   try:
-    if not fs.do_as_user(django_username_short, fs.get_home_dir):
+    if not fs:
+      LOG.info('No fs configured, skipping home directory creation for user: %s' % django_username_short)
+    elif not fs.do_as_user(django_username_short, fs.get_home_dir):
       fs.do_as_user(django_username_short, fs.create_home_dir)
       LOG.info('Created home directory for user: %s' % django_username_short)
     else:

+ 11 - 0
desktop/core/src/desktop/lib/connectors/models.py

@@ -52,6 +52,17 @@ class BaseConnector(models.Model):
   def __str__(self):
     return '%s (%s)' % (self.name, self.dialect)
 
+  def to_dict(self):
+    return {
+      'id': self.id,
+      'type': str(self.id),
+      'name': self.name,
+      'description': self.description,
+      'dialect': self.dialect,
+      'settings': self.settings,
+      'last_modified': self.last_modified
+    }
+
 
 if ENABLE_ORGANIZATIONS.get():
   class ConnectorManager(models.Manager):

+ 5 - 1
desktop/core/src/desktop/models.py

@@ -96,7 +96,11 @@ def _version_from_properties(f):
 
 def get_sample_user_install(user):
   if ENABLE_ORGANIZATIONS.get():
-   return SAMPLE_USER_INSTALL + '@' + get_organization(email=user.email).domain
+    organization = get_organization(email=user.email)
+    if organization.is_multi_user:
+      return SAMPLE_USER_INSTALL + '@' + organization.domain
+    else:
+      return organization.domain
   else:
     return SAMPLE_USER_INSTALL
 

+ 17 - 6
desktop/libs/notebook/src/notebook/api.py

@@ -791,7 +791,8 @@ def export_result(request):
       response = task.execute(request)
     else:
       notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
-      response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=save_as_table&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
+      response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=save_as_table&notebook=' + str(notebook_id) + \
+          '&snippet=0&destination=' + destination
       response['status'] = 0
     request.audit = {
       'operation': 'EXPORT',
@@ -820,7 +821,8 @@ def export_result(request):
       response = task.execute(request)
     else:
       notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
-      response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=insert_as_query&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
+      response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=insert_as_query&notebook=' + str(notebook_id) + \
+          '&snippet=0&destination=' + destination
       response['status'] = 0
     request.audit = {
       'operation': 'EXPORT',
@@ -834,7 +836,10 @@ def export_result(request):
 
       if data_format == 'dashboard':
         engine = notebook['type'].replace('query-', '')
-        response['watch_url'] = reverse('dashboard:browse', kwargs={'name': notebook_id}) + '?source=query&engine=%(engine)s' % {'engine': engine}
+        response['watch_url'] = reverse(
+            'dashboard:browse',
+            kwargs={'name': notebook_id}
+        ) + '?source=query&engine=%(engine)s' % {'engine': engine}
         response['status'] = 0
       else:
         sample = get_api(request, snippet).fetch_result(notebook, snippet, rows=4, start_over=True)
@@ -853,11 +858,12 @@ def export_result(request):
         ]
     else:
       notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
-      response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=index_query&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
+      response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=index_query&notebook=' + str(notebook_id) + \
+          '&snippet=0&destination=' + destination
       response['status'] = 0
 
     if response.get('status') != 0:
-      response['message'] =  _('Exporting result failed.')
+      response['message'] = _('Exporting result failed.')
 
   return JsonResponse(response)
 
@@ -892,7 +898,12 @@ def statement_compatibility(request):
 
   api = get_api(request, snippet)
 
-  response['query_compatibility'] = api.statement_compatibility(notebook, snippet, source_platform=source_platform, target_platform=target_platform)
+  response['query_compatibility'] = api.statement_compatibility(
+      notebook,
+      snippet,
+      source_platform=source_platform,
+      target_platform=target_platform
+  )
   response['status'] = 0
 
   return JsonResponse(response)

+ 2 - 2
desktop/libs/notebook/src/notebook/models.py

@@ -239,13 +239,13 @@ class MockedDjangoRequest(object):
     self.method = method
 
 
-def import_saved_beeswax_query(bquery):
+def import_saved_beeswax_query(bquery, interpreter=None):
   design = bquery.get_design()
 
   return make_notebook(
       name=bquery.name,
       description=bquery.desc,
-      editor_type=_convert_type(bquery.type, bquery.data),
+      editor_type=interpreter['type'] if interpreter else _convert_type(bquery.type, bquery.data),
       statement=design.hql_query,
       status='ready',
       files=design.file_resources,

+ 12 - 3
desktop/libs/notebook/src/notebook/views.py

@@ -26,8 +26,11 @@ from django.utils.translation import ugettext as _
 from django.views.decorators.clickjacking import xframe_options_exempt
 
 from beeswax.data_export import DOWNLOAD_COOKIE_AGE
+from beeswax.management.commands import beeswax_install_examples
+from desktop.auth.decorators import admin_required
 from desktop.conf import ENABLE_DOWNLOAD, USE_NEW_EDITOR
 from desktop.lib import export_csvxls
+from desktop.lib.connectors.models import Connector
 from desktop.lib.django_util import render, JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.json_utils import JSONEncoderForHTML
@@ -36,7 +39,7 @@ from desktop.views import serve_403_error
 from metadata.conf import has_optimizer, has_catalog, has_workload_analytics
 
 from notebook.conf import get_ordered_interpreters, SHOW_NOTEBOOKS
-from notebook.connectors.base import Notebook, _get_snippet_name
+from notebook.connectors.base import Notebook, _get_snippet_name, get_interpreter
 from notebook.connectors.spark_shell import SparkApi
 from notebook.decorators import check_editor_access_permission, check_document_access_permission, check_document_modify_permission
 from notebook.management.commands.notebook_setup import Command
@@ -371,13 +374,19 @@ def download(request):
   return response
 
 
+@admin_required
 def install_examples(request):
   response = {'status': -1, 'message': ''}
 
   if request.method == 'POST':
     try:
-      if request.POST.get('dialect') == 'hive':
-        pass
+      connector = Connector.objects.get(id=request.POST.get('connector'))
+      if connector:
+        app_name = 'beeswax' if connector.dialect == 'hive' else 'impala'
+        db_name = request.POST.get('db_name', 'default')
+        interpreter = get_interpreter(connector_type=connector.to_dict()['type'], user=request.user)
+
+        beeswax_install_examples.Command().handle(app_name=app_name, db_name=db_name, user=request.user, interpreter=interpreter)
       else:
         Command().handle(user=request.user)
         response['status'] = 0