浏览代码

HUE-8591 [cluster] Hook in remote Impala coordinator URL of selected cluster

Romain Rigaux 7 年之前
父节点
当前提交
605c500

+ 4 - 1
apps/beeswax/src/beeswax/server/dbms.py

@@ -78,7 +78,10 @@ def get(user, query_server=None, cluster=None):
 
 def get_query_server_config(name='beeswax', server=None, cluster=None):
   if cluster and cluster != CLUSTER_ID.get():
-    cluster_config = Cluster(user=None).get_config(cluster)
+    if cluster and 'altus:dataware:k8s' in cluster:
+      cluster_config = {'server_host': 'impala-coordinatortim', 'name': cluster}
+    else:
+      cluster_config = Cluster(user=None).get_config(cluster)
   else:
     cluster_config = None
 

+ 1 - 1
apps/impala/src/impala/dbms.py

@@ -35,7 +35,7 @@ def get_query_server_config(cluster_config=None):
   query_server = {
         'server_name': 'impala' + ('-' + cluster_config.get('name') if cluster_config else ''),
         'server_host': conf.SERVER_HOST.get() if not cluster_config else cluster_config.get('server_host'),
-        'server_port': conf.SERVER_PORT.get(),
+        'server_port': 21000, #conf.SERVER_PORT.get(),
         'principal': conf.IMPALA_PRINCIPAL.get(),
         'impersonation_enabled': conf.IMPERSONATION_ENABLED.get(),
         'querycache_rows': conf.QUERYCACHE_ROWS.get(),

+ 19 - 7
desktop/core/src/desktop/api2.py

@@ -35,17 +35,17 @@ from django.views.decorators.http import require_POST
 
 from metadata.conf import has_navigator
 from metadata.catalog_api import search_entities as metadata_search_entities, _highlight, search_entities_interactive as metadata_search_entities_interactive
-from notebook.connectors.altus import SdxApi, AnalyticDbApi, DataEngApi
+from notebook.connectors.altus import SdxApi, AnalyticDbApi, DataEngApi, DataWarehouse2Api
 from notebook.connectors.base import Notebook
 from notebook.views import upgrade_session_properties
 
 from desktop.lib.django_util import JsonResponse
+from desktop.conf import get_clusters, IS_K8_ONLY
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.export_csvxls import make_response
 from desktop.lib.i18n import smart_str, force_unicode
 from desktop.models import Document2, Document, Directory, FilesystemException, uuid_default, \
   UserPreferences, get_user_preferences, set_user_preferences, get_cluster_config
-from desktop.conf import get_clusters
 
 
 LOG = logging.getLogger(__name__)
@@ -96,7 +96,10 @@ def get_context_namespaces(request, interface):
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
       # Note: attaching computes to namespaces might be done via the frontend in the future
       if interface == 'impala':
-        adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters']
+        if IS_K8_ONLY.get():
+          adb_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters']
+        else:
+          adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters']
         for _cluster in adb_clusters: # Add "fake" namespace if needed
           if not _cluster.get('namespaceCrn'):
             _cluster['namespaceCrn'] = _cluster['crn']
@@ -106,16 +109,20 @@ def get_context_namespaces(request, interface):
       else:
         adb_clusters = []
 
-      sdx_namespaces = SdxApi(request.user).list_namespaces()
+      if IS_K8_ONLY.get():
+        sdx_namespaces = []
+      else:
+        sdx_namespaces = SdxApi(request.user).list_namespaces()
+
       # Adding "fake" namespace for cluster without one
-      sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn')])
+      sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or IS_K8_ONLY.get()])
 
       namespaces.extend([{
           'id': namespace.get('crn', 'None'),
           'name': namespace.get('namespaceName'),
           'status': namespace.get('status'),
           'computes': [_cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn')]
-        } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED'
+        } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED' or IS_K8_ONLY.get()
       ])
 
   response[interface] = namespaces
@@ -142,6 +149,11 @@ def get_context_computes(request, interface):
     ])
 
   if interface == 'impala' or interface == 'report':
+    if IS_K8_ONLY.get():
+      dw_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters']
+    else:
+      dw_clusters = AnalyticDbApi(request.user).list_clusters()['clusters']
+
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
       computes.extend([{
           'id': cluster.get('crn'),
@@ -149,7 +161,7 @@ def get_context_computes(request, interface):
           'status': cluster.get('status'),
           'namespace': cluster.get('namespaceCrn', cluster.get('crn')),
           'type': 'altus-dw'
-        } for cluster in AnalyticDbApi(request.user).list_clusters()['clusters'] if cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515']
+        } for cluster in dw_clusters if (cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515') or IS_K8_ONLY.get()]
       )
 
   if interface == 'oozie' or interface == 'spark2':

+ 8 - 0
desktop/core/src/desktop/conf.py

@@ -1600,6 +1600,14 @@ IS_MULTICLUSTER_ONLY = Config(
   help=_('Choose whether to pick configs only from [desktop] [[cluster]]')
 )
 
+IS_K8_ONLY = Config(
+  key='is_k8_only',
+  default=True,
+  type=coerce_bool,
+  help=_('Choose whether to pick configs only from [desktop] [[cluster]]')
+)
+
+
 def get_clusters(user):
   clusters = []
 

+ 5 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -319,10 +319,14 @@ def get_api(request, snippet):
     cluster = None
   if not cluster and snippet.get('compute'): # Via notebook.ko.js
     cluster = snippet.get('compute').get('id')
-  if cluster and 'crn:altus:dataware:' in cluster:
+
+  if cluster and 'altus:dataware:k8s' in cluster:
+    interface = 'hiveserver2'
+  elif cluster and 'crn:altus:dataware:' in cluster:
     interface = 'altus-adb'
   elif cluster and 'crn:altus:dataeng:' in cluster:
     interface = 'dataeng'
+
   if cluster:
     LOG.info('Selected cluster %s' % cluster)