Browse Source

HUE-8330 [core] Skeleton of API to provide context of namespaces/compute for Hive

Hive only and list of clusters harcoded in the UI in metastore.model.js.

$.get("/desktop/api2/context/editor/hive", function(data) {
  console.log(ko.mapping.toJSON(data));
});

$.get("/desktop/api2/context/browser/hive", function(data) {
  console.log(ko.mapping.toJSON(data));
});

In hue.ini:

[desktop]
[[clusters]]

    [[[Default]]]
      name=default
      server_host=localhost

    [[[nightlyc6]]]
      name=nightly c6
      server_host=nightly6x-unsecure-1.gce.cloudera.com

    [[[selfserviceanalytics]]]
      name=self service analytics
      server_host=self-service-analytics-1.gce.cloudera.com
Romain Rigaux 7 years ago
parent
commit
237ba8e359

+ 4 - 4
apps/beeswax/src/beeswax/api.py

@@ -126,7 +126,7 @@ def _autocomplete(db, database=None, table=None, column=None, nested=None, query
 
       if table.is_impala_only: # Expand Kudu columns information
         query_server = get_query_server_config('impala')
-        db = dbms.get(db.client.user, query_server)
+        db = dbms.get(db.client.user, query_server, cluster=db.cluster)
 
         col_options = db.get_table_describe(database, table.name)
         extra_col_options = dict([(col[0], dict(zip(col_options.cols(), col))) for col in col_options.rows()])
@@ -861,10 +861,10 @@ def close_session(request, session_id):
 
 
 # Proxy API for Metastore App
-def describe_table(request, database, table):
+def describe_table(request, database, table, cluster=None):
   try:
-    from metastore.views import describe_table
-    return describe_table(request, database, table)
+    from metastore.views import describe_table as metastore_describe_table
+    return metastore_describe_table(request, database, table, cluster=cluster)
   except Exception, e:
     LOG.exception('Describe table failed')
     raise PopupException(_('Problem accessing table metadata'), detail=e)

+ 10 - 12
apps/beeswax/src/beeswax/server/dbms.py

@@ -27,8 +27,8 @@ from django.utils.translation import ugettext as _
 from desktop.lib.django_util import format_preserving_redirect
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.parameterization import substitute_variables
-from desktop.models import Cluster, ANALYTIC_DB
 from desktop.lib.view_util import location_to_url
+from desktop.models import Cluster
 from indexer.file_format import HiveFormat
 
 from beeswax import hive_site
@@ -47,17 +47,12 @@ DBMS_CACHE = {}
 DBMS_CACHE_LOCK = threading.Lock()
 
 
-def get(user, query_server=None):
+def get(user, query_server=None, cluster=None):
   global DBMS_CACHE
   global DBMS_CACHE_LOCK
 
   if query_server is None:
-    cluster_type = Cluster(user).get_type()
-    if cluster_type == ANALYTIC_DB:
-      kwargs = {'name': 'impala'}
-    else:
-      kwargs = {}
-    query_server = get_query_server_config(**kwargs)
+    query_server = get_query_server_config(cluster=cluster)
 
   DBMS_CACHE_LOCK.acquire()
   try:
@@ -80,16 +75,19 @@ def get(user, query_server=None):
     DBMS_CACHE_LOCK.release()
 
 
-def get_query_server_config(name='beeswax', server=None):
+def get_query_server_config(name='beeswax', server=None, cluster=None):
+  if cluster:
+    cluster_config = Cluster(user=None).get_config(cluster)
+
   if name == 'impala':
     from impala.dbms import get_query_server_config as impala_query_server_config
-    query_server = impala_query_server_config()
+    query_server = impala_query_server_config(cluster_config=cluster_config)
   else:
     kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
 
     query_server = {
-        'server_name': name,
-        'server_host': HIVE_SERVER_HOST.get(),
+        'server_name': name if not cluster else '%s-%s' % (name, cluster_config.get('id')),
+        'server_host': HIVE_SERVER_HOST.get() if not cluster else cluster_config.get('server_host'),
         'server_port': HIVE_SERVER_PORT.get(),
         'principal': kerberos_principal,
         'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {

+ 1 - 3
apps/beeswax/src/beeswax/urls.py

@@ -44,19 +44,16 @@ urlpatterns = [
 ]
 
 urlpatterns += [
-
   url(r'^create/database$', beeswax_create_database.create_database, name='create_database'),
 ]
 
 urlpatterns += [
-
   url(r'^create/create_table/(?P<database>\w+)$', beeswax_create_table.create_table, name='create_table'),
   url(r'^create/import_wizard/(?P<database>\w+)$', beeswax_create_table.import_wizard, name='import_wizard'),
   url(r'^create/auto_load/(?P<database>\w+)$', beeswax_create_table.load_after_create, name='load_after_create'),
 ]
 
 urlpatterns += [
-
   url(r'^api/session/?$', beeswax_api.get_session, name='api_get_session'),
   url(r'^api/session/(?P<session_id>\d+)/?$', beeswax_api.get_session, name='api_get_session'),
   url(r'^api/session/(?P<session_id>\d+)/close/?$', beeswax_api.close_session, name='api_close_session'),
@@ -86,6 +83,7 @@ urlpatterns += [
   url(r'^api/query/clear_history/?$', beeswax_api.clear_history, name='clear_history'),
 
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/?$', beeswax_api.describe_table, name='describe_table'),
+  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/(?P<cluster>.+)$', beeswax_api.describe_table, name='describe_table'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/indexes/?$', beeswax_api.get_indexes, name='get_indexes'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/sample/?$', beeswax_api.get_sample_data, name='get_sample_data'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/sample/?$', beeswax_api.get_sample_data, name='get_sample_data_column'),

+ 0 - 2
apps/filebrowser/src/filebrowser/views.py

@@ -342,8 +342,6 @@ def listdir(request, path):
     Implements directory listing (or index).
 
     Intended to be called via view().
-
-    TODO: Remove?
     """
     if not request.fs.isdir(path):
         raise PopupException(_("Not a directory: %(path)s") % {'path': path})

+ 3 - 3
apps/impala/src/impala/dbms.py

@@ -31,10 +31,10 @@ from impala import conf
 LOG = logging.getLogger(__name__)
 
 
-def get_query_server_config():
+def get_query_server_config(cluster_config=None):
   query_server = {
-        'server_name': 'impala',
-        'server_host': conf.SERVER_HOST.get(),
+        'server_name': 'impala' + ('-' + cluster_config.get('name') if cluster_config else ''),
+        'server_host': conf.SERVER_HOST.get() if not cluster_config else cluster_config.get('server_host'),
         'server_port': conf.SERVER_PORT.get(),
         'principal': conf.IMPALA_PRINCIPAL.get(),
         'impersonation_enabled': conf.IMPERSONATION_ENABLED.get(),

+ 7 - 0
apps/metastore/src/metastore/static/metastore/js/metastore.model.js

@@ -35,6 +35,13 @@ var MetastoreDatabase = (function () {
     self.loadingTableComments = ko.observable(false);
     self.loadingTablePopularity = ko.observable(false);
 
+
+    self.contexts = ko.observableArray(['Default', 'nightlyc6', 'selfserviceanalytics']);
+    self.context = ko.observable('Default');
+    self.context.subscribe(function(newVal) {
+      window.context = newVal;
+    });
+
     self.tables = ko.observableArray();
 
     self.loading = ko.pureComputed(function () {

+ 1 - 0
apps/metastore/src/metastore/templates/metastore.mako

@@ -1005,6 +1005,7 @@ ${ components.menubar(is_embeddable) }
               <a href="javascript: void(0);" class="btn btn-default" data-bind="click: reload" title="${_('Refresh the table')}"><i class="fa fa-refresh" data-bind="css: { 'fa-spin blue' : refreshing }"></i> ${_('Refresh')}</a>
               <!-- /ko -->
               <!-- ko if: !table() -->
+              <select data-bind="selectize: contexts, value: context" class="input-medium"></select>
               <a href="javascript: void(0);" class="btn btn-default" data-bind="click: reload" title="${_('Refresh the database')}"><i class="fa fa-refresh" data-bind="css: { 'fa-spin blue' : refreshing }"></i> ${_('Refresh')}</a>
               <!-- /ko -->
               <!-- /ko -->

+ 2 - 0
apps/metastore/src/metastore/urls.py

@@ -25,12 +25,14 @@ urlpatterns = [
   url(r'^databases/drop/?$', metastore_views.drop_database, name='drop_database'),
   url(r'^databases/(?P<database>\w+)/alter$', metastore_views.alter_database, name='alter_database'),
   url(r'^databases/(?P<database>\w+)/metadata$', metastore_views.get_database_metadata, name='get_database_metadata'),
+  url(r'^databases/(?P<database>\w+)/metadata/(?P<cluster>.+)$', metastore_views.get_database_metadata, name='get_database_metadata'),
 
   url(r'^tables/(?P<database>\w+)?/?$', metastore_views.show_tables, name='show_tables'),
   url(r'^tables/drop/(?P<database>\w+)$', metastore_views.drop_table, name='drop_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/?$', metastore_views.describe_table, name='describe_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/alter$', metastore_views.alter_table, name='alter_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/metadata$', metastore_views.get_table_metadata, name='get_table_metadata'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/metadata/(?P<cluster>.+)$', metastore_views.get_table_metadata, name='get_table_metadata'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/load$', metastore_views.load_table, name='load_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/read$', metastore_views.read_table, name='read_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/queries$', metastore_views.table_queries, name='table_queries'),

+ 8 - 7
apps/metastore/src/metastore/views.py

@@ -36,7 +36,6 @@ from beeswax.models import SavedQuery
 from beeswax.server import dbms
 from beeswax.server.dbms import get_query_server_config
 from desktop.lib.view_util import location_to_url
-from urlparse import urlparse
 from metadata.conf import has_optimizer, has_navigator, get_optimizer_url, get_navigator_url
 from notebook.connectors.base import Notebook, QueryError
 from notebook.models import make_notebook
@@ -158,10 +157,10 @@ def alter_database(request, database):
   return JsonResponse(response)
 
 
-def get_database_metadata(request, database):
+def get_database_metadata(request, database, cluster=None):
   response = {'status': -1, 'data': ''}
   source_type = request.POST.get('source_type', 'hive')
-  db = _get_db(user=request.user, source_type=source_type)
+  db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
 
   try:
     db_metadata = db.get_database(database)
@@ -265,9 +264,9 @@ def get_table_metadata(request, database, table):
   return JsonResponse(response)
 
 
-def describe_table(request, database, table):
+def describe_table(request, database, table, cluster=None):
   app_name = get_app_name(request)
-  db = _get_db(user=request.user)
+  db = _get_db(user=request.user, cluster=cluster)
 
   try:
     table = db.get_table(database, table)
@@ -659,7 +658,7 @@ def has_write_access(user):
 
 
 
-def _get_db(user, source_type=None):
+def _get_db(user, source_type=None, cluster=None):
   if source_type is None:
     cluster_config = get_cluster_config(user)
     if FORCE_HS2_METADATA.get() and cluster_config['app_config'].get('editor') and 'hive' in cluster_config['app_config'].get('editor')['interpreter_names']:
@@ -667,7 +666,9 @@ def _get_db(user, source_type=None):
     else:
       source_type = cluster_config['default_sql_interpreter']
 
-  query_server = get_query_server_config(name=source_type if source_type != 'hive' else 'beeswax')
+  name = source_type if source_type != 'hive' else 'beeswax'
+
+  query_server = get_query_server_config(name=name, cluster=cluster)
   return dbms.get(user, query_server)
 
 

+ 35 - 0
desktop/core/src/desktop/api2.py

@@ -45,6 +45,7 @@ from desktop.lib.export_csvxls import make_response
 from desktop.lib.i18n import smart_str, force_unicode
 from desktop.models import Document2, Document, Directory, FilesystemException, uuid_default, \
   UserPreferences, get_user_preferences, set_user_preferences, get_cluster_config
+from desktop.conf import get_clusters
 
 
 LOG = logging.getLogger(__name__)
@@ -75,6 +76,40 @@ def get_config(request):
   return JsonResponse(config)
 
 
+@api_error_handler
+def get_context(request, app, engine):
+  context = {}
+
+  if app == 'editor':
+    if engine == 'hive':
+      context['editor'] = {
+        'hive': {
+          'computes': [
+            {
+              'id': cluster['id'],
+              'name': cluster['name']
+            } for cluster in get_clusters().values()
+          ]
+        }
+      }
+  elif app == 'browser':
+    if engine == 'hive':
+      context['browser'] = {
+        'hive': {
+          'namespaces': [
+            {
+              'id': cluster['id'],
+              'name': cluster['name']
+            } for cluster in get_clusters().values()
+          ]
+        }
+      }
+
+  context['status'] = 0
+
+  return JsonResponse(context)
+
+
 @api_error_handler
 def search_documents(request):
   """

+ 21 - 17
desktop/core/src/desktop/conf.py

@@ -1590,37 +1590,41 @@ def get_clusters():
     cluster_config = CLUSTERS.get()
     clusters = OrderedDict([
       (i, {
-        'name': i,
+        'id': i,
+        'name': cluster_config[i].NAME.get() or i,
         'type': cluster_config[i].TYPE.get(),
-        'interfaces': [
-            {'name': i, 'type': cluster_config[i].TYPE.get(), 'interface': interface}
-            for interface in cluster_config[i].INTERFACES.get()
-        ]
-      }) for i in cluster_config]
-    )
+        'server_host': cluster_config[i].SERVER_HOST.get()
+      }) for i in cluster_config
+    ])
   else:
-    clusters = OrderedDict([('Default', {'name': 'Default', 'type': 'ini', 'interfaces': []})])
+    clusters = OrderedDict([('Default', {'name': 'Default', 'type': 'ini'})])
 
   return clusters
 
 
 CLUSTERS = UnspecifiedConfigSection(
   "clusters",
-  help="One entry for each additional cluster Hue can interact with.",
+  help="One entry for each additional remote cluster Hue can interact with.",
   each=ConfigSection(
-    help=_("Name of the cluster to show to the user."),
+    help=_("Id of the cluster."),
     members=dict(
+      NAME=Config(
+          "name",
+          help=_("Nice name of the cluster to show to the user. Same as id if not specified."),
+          default=None,
+          type=str,
+      ),
       TYPE=Config(
           "type",
-          help=_("Type of cluster, e.g. local ini, CM API, Dataeng, Arcus, BigQuery, Presto."),
-          default='local',
+          help=_("Type of cluster, e.g. direct, local ini, CM API, Dataeng, Arcus, BigQuery, Presto."),
+          default='direct',
           type=str,
       ),
-      INTERFACES=Config(
-          "interfaces",
-          help=_("List of cluster instances of the cluster (optional)."),
-          default=[],
-          type=coerce_csv,
+      SERVER_HOST=Config(
+          "server_host",
+          help=_("The host service to contact."),
+          default=None,
+          type=str,
       ),
     )
   )

+ 12 - 2
desktop/core/src/desktop/models.py

@@ -1562,7 +1562,13 @@ ANALYTIC_DB = 'analyticdb'
 
 
 class ClusterConfig():
-
+  """
+  Configuration of the apps and engines that each individual user sees on the core Hue.
+  Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
+  
+  TODO: rename to HueConfig
+  TODO: get list of contexts dynamically
+  """
   def __init__(self, user, apps=None, cluster_type='ini'):
     self.user = user
     self.apps = appmanager.get_apps_dict(self.user) if apps is None else apps
@@ -1906,11 +1912,15 @@ class Cluster():
 
   def __init__(self, user):
     self.user = user
-    self.data = get_clusters()['Default']
+    self.clusters = get_clusters()
+    self.data = self.clusters['Default']
 
   def get_type(self):
     return self.data['type']
 
+  def get_config(self, name):
+    return self.clusters[name]
+
 
 def _get_apps(user, section=None):
   current_app = None

+ 7 - 2
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -1293,8 +1293,9 @@ var ApiHelper = (function () {
         notebook: {},
         snippet: ko.mapping.toJSON({
           type: sourceType,
-          source: isQuery ? 'query' : 'data'
-        })
+          source: isQuery ? 'query' : 'data',
+        }),
+        cluster: ko.mapping.toJSON(window.context || '')
       },
       timeout: options.timeout
     }).success(function (data) {
@@ -1391,6 +1392,10 @@ var ApiHelper = (function () {
       }
     }
 
+    if (window.context) {
+      url += '/' + window.context;
+    }
+
     var request = self.simpleGet(url, {
       'format' : 'json'
     }, {

+ 1 - 0
desktop/core/src/desktop/urls.py

@@ -155,6 +155,7 @@ dynamic_patterns += [
   url(r'^desktop/api2/doc/share/?$', desktop_api2.share_document),
 
   url(r'^desktop/api2/get_config/?$', desktop_api2.get_config),
+  url(r'^desktop/api2/context/(?P<app>\w+)/(?P<engine>\w+)/?$', desktop_api2.get_context),
   url(r'^desktop/api2/user_preferences/(?P<key>\w+)?$', desktop_api2.user_preferences, name="desktop.api2.user_preferences"),
 
   url(r'^desktop/api2/doc/export/?$', desktop_api2.export_documents),

+ 4 - 5
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -279,13 +279,11 @@ def get_api(request, snippet):
   interface = interpreter['interface']
 
   # Multi cluster
-  cluster = Cluster(request.user)
-  if cluster and cluster.get_type() == 'dataeng':
-    interface = 'dataeng'
+  cluster = json.loads(request.POST.get('cluster', '""'))
 
   if interface == 'hiveserver2':
     from notebook.connectors.hiveserver2 import HS2Api
-    return HS2Api(user=request.user, request=request)
+    return HS2Api(user=request.user, request=request, cluster=cluster)
   elif interface == 'oozie':
     return OozieApi(user=request.user, request=request)
   elif interface == 'livy':
@@ -333,10 +331,11 @@ def _get_snippet_session(notebook, snippet):
 
 class Api(object):
 
-  def __init__(self, user, interpreter=None, request=None):
+  def __init__(self, user, interpreter=None, request=None, cluster=None):
     self.user = user
     self.interpreter = interpreter
     self.request = request
+    self.cluster = cluster
 
   def create_session(self, lang, properties=None):
     return {

+ 3 - 3
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -444,7 +444,7 @@ class HS2Api(Api):
 
   @query_error_handler
   def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
-    db = self._get_db(snippet)
+    db = self._get_db(snippet, cluster=self.cluster)
     query = None
 
     if snippet.get('query'):
@@ -764,7 +764,7 @@ DROP TABLE IF EXISTS `%(table)s`;
     return HiveServerQueryHandle(**snippet['result']['handle'])
 
 
-  def _get_db(self, snippet, async=False):
+  def _get_db(self, snippet, async=False, cluster=None):
     if not async and snippet['type'] == 'hive':
       name = 'beeswax'
     elif snippet['type'] == 'hive':
@@ -774,7 +774,7 @@ DROP TABLE IF EXISTS `%(table)s`;
     else:
       name = 'sparksql'
 
-    return dbms.get(self.user, query_server=get_query_server_config(name=name))
+    return dbms.get(self.user, query_server=get_query_server_config(name=name, cluster=cluster))
 
 
   def _parse_job_counters(self, job_id):