Browse Source

HUE-8330 [core] Get namespace and compute from altus

Romain Rigaux 7 years ago
parent
commit
e56d72757c

+ 1 - 1
apps/metastore/src/metastore/templates/metastore.mako

@@ -987,7 +987,7 @@ ${ components.menubar(is_embeddable) }
               <!-- ko component: {
                 name: 'hue-drop-down',
                 params: {
-                  icon: 'fa-snowflake-o',
+                  icon: 'fa-server',
                   value: activeSourceContext,
                   entries: sourceContexts,
                   searchable: true,

+ 43 - 13
desktop/core/src/desktop/api2.py

@@ -36,6 +36,7 @@ from django.views.decorators.http import require_POST
 from metadata.conf import has_navigator
 from metadata.catalog_api import search_entities as metadata_search_entities, _highlight
 from metadata.catalog_api import search_entities_interactive as metadata_search_entities_interactive
+from notebook.connectors.altus import SdxApi, AnalyticDbApi
 from notebook.connectors.base import Notebook
 from notebook.views import upgrade_session_properties
 
@@ -78,39 +79,68 @@ def get_config(request):
 
 @api_error_handler
 def get_context_namespaces(request, interface):
-  namespaces = {}
+  response = {}
+  namespaces = []
 
   clusters = get_clusters(request.user).values()
 
   if interface == 'hive':
-    namespaces['hive'] = [{
+    namespaces.extend([{
         'id': cluster['id'],
         'name': cluster['name']
-      } for cluster in clusters
-    ]
-
-  namespaces['status'] = 0
+      } for cluster in clusters if cluster.get('type') == 'direct' # and interfaces == 'hive'
+    ])
+
+    # From Altus SDX
+    if [cluster for cluster in clusters if cluster['type'] == 'altus']:
+      namespaces.extend([{"status": "CREATED", "id": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:doc_test/b0a5c594-0e4f-4084-ab00-bde55b4ab03e", "name": "doc_test"}, {"status": "CREATED", "id": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:mschumpert/be6ae3c5-1790-468c-be6e-0d61a47d066d", "name": "mschumpert"}, {"status": "CREATED", "id": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:cca-altus2-ns/bffb9ade-cbf5-4b47-8ac0-3b67b5f2472d", "name": "cca-altus2-ns"}, {"status": "CREATED", "id": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:cca-sdx-ns/ce4b569b-a8e1-459d-a12d-8ae671f2db4f", "name": "cca-sdx-ns"}, {"status": "CREATED", "id": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:cca-altus1-ns/e0fb5b22-5cbc-445e-86f3-959e4aee2b66", "name": "cca-altus1-ns"}, {"status": "CREATED", "id": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:victor-namespace/ea714d02-0bed-425e-a09b-556367e559b0", "name": "victor-namespace"}, {"status": "CREATED", "id": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:BP_Enterprise/ecd8a000-4671-4bd0-9b66-b939aabb5a76", "name": "BP_Enterprise"}])
+# Js error for some reason
+#       namespaces.extend([{
+#           'id': namespace.get('crn', 'None'),
+#           'name': namespace.get('namespaceName', 'Unknown'),
+#           'status': namespace.get('status'),
+#           # creationDate
+#         } for namespace in SdxApi(request.user).list_namespaces()]
+#       )
+
+  response[interface] = namespaces
+  response['status'] = 0
 
-  return JsonResponse(namespaces)
+  return JsonResponse(response)
 
 
 @api_error_handler
 def get_context_computes(request, interface):
-  computes = {}
+  response = {}
+  computes = []
 
   clusters = get_clusters(request.user).values()
 
   if interface == 'hive':
-    computes['hive'] = [{
+    computes.extend([{
         'id': cluster['id'],
         'name': cluster['name'],
         'namespace': cluster['id'] # Dummy
       } for cluster in clusters
-    ]
+    ])
+
+    # From Altus
+    if [cluster for cluster in clusters if cluster['type'] == 'altus']:
+      computes.extend([{
+          'id': cluster.get('crn', 'None'),
+          'name': cluster.get('clusterName', 'Unknown'),
+          'status': cluster.get('status'),
+          # namespaceCrn
+          # environmentType
+          # secured
+          # cdhVersion
+        } for cluster in AnalyticDbApi(request.user).list_clusters()]
+      )
+
+  response[interface] = computes
+  response['status'] = 0
 
-  computes['status'] = 0
-
-  return JsonResponse(computes)
+  return JsonResponse(response)
 
 
 @api_error_handler

+ 14 - 12
desktop/core/src/desktop/conf.py

@@ -1586,31 +1586,33 @@ DISABLE_HUE_3 = Config( # To remove in Hue 5
 
 
 def get_clusters(user):
-  # Get additional remote multi clusters
-  cluster_config = CLUSTERS.get()
-  clusters = [
-    (i, {
-      'id': i,
-      'name': cluster_config[i].NAME.get() or i,
-      'type': cluster_config[i].TYPE.get(),
-      'server_host': cluster_config[i].SERVER_HOST.get()
-    }) for i in cluster_config
-  ]
+  clusters = []
 
   # Get core standalone config if there
   apps = appmanager.get_apps_dict(user)
   if 'beeswax' in apps:
     from beeswax.conf import HIVE_SERVER_HOST
-    clusters.insert(0,
+    clusters.append(
       (CLUSTER_ID.get(), {
         'id': CLUSTER_ID.get(),
         'name': CLUSTER_ID.get(),
-        'type': 'single',
+        'type': 'direct',
         'server_host': HIVE_SERVER_HOST.get()
         }
       )
     )
 
+  # Get additional remote multi clusters
+  cluster_config = CLUSTERS.get()
+  clusters.extend([
+    (i, {
+      'id': i,
+      'name': cluster_config[i].NAME.get() or i,
+      'type': cluster_config[i].TYPE.get(),
+      'server_host': cluster_config[i].SERVER_HOST.get()
+    }) for i in cluster_config
+  ])
+
   return OrderedDict(clusters)
 
 

+ 99 - 0
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import json
+import subprocess
+
+from django.urls import reverse
+from django.utils.translation import ugettext as _
+
+from desktop.lib.exceptions_renderable import PopupException
+
+
+LOG = logging.getLogger(__name__)
+
+
+def _exec(command, args):
+  try:
+    data = subprocess.check_output([
+        'altus',
+        command,
+       ] +
+       args
+    )
+  except Exception, e:
+    raise PopupException(e, title=_('Error accessing'))
+
+  response = json.loads(data)
+
+  return response
+
+
+class IAMApi(): pass
+# altus iam list-user-assigned-roles --user=crn:altus:ia
+
+
+class SdxApi():
+
+  def __init__(self, user): pass
+
+  def list_namespaces(self):
+    """
+    e.g. returns
+    [{
+      u'status': u'CREATED',
+      u'namespaceName': u'cca-5150-ns',
+      u'creationDate': u'2018-06-03T23:24:46.125000+00:00',
+      u'crn': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:cca-5150-ns/f54461af-b241-4f1d-a521-ab3624e841c4'},
+      ...
+    ]
+    """
+
+    args = ['list-namespaces']
+    return _exec('sdx', args)['namespaces']
+
+
+class AnalyticDbApi():
+
+  def __init__(self, user): pass
+
+  def list_clusters(self):
+    """
+    e.g. returns
+    [{
+            "status": "CREATED",
+            "namespaceCrn": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:spot-ns/7bdb225f-a7a1-408e-8503-1b3a422cc039",
+            "workersGroupSize": 4,
+            "clusterName": "spot",
+            "environmentType": "AWS",
+            "secured": false,
+            "environmentCrn": "crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:Spot-AWS-dev2/5a6d0ced-c8af-4fa3-9b24-8c5a3ea11cf8",
+            "securityConfiguration": {
+                "enabled": false
+            },
+            "creationDate": "2018-06-01T13:14:43.530000+00:00",
+            "crn": "crn:altus:analyticdb:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:cluster:spot/70595482-6a46-4a9d-b395-56fcabe079e4",
+            "instanceType": "r4.4xlarge",
+            "cdhVersion": "CDH514"
+        },
+      ...
+    ]
+    """
+
+    args = ['list-clusters']
+    return _exec('analyticdb', args)['clusters']

+ 1 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -357,7 +357,7 @@ var EditorViewModel = (function() {
     };
 
     self.computes = ko.observableArray(['default', 'nightlyc6', 'selfserviceanalytics']);
-    self.selectedCompute = ko.observable(typeof snippet.selectedCompute != "undefined" && snippet.selectedCompute != null ? snippet.selectedCompute : null);
+    self.selectedCompute = ko.observable(typeof snippet.selectedCompute != "undefined" && snippet.selectedCompute != null ? snippet.selectedCompute : self.computes().length > 0 ? self.computes()[0] : null);
 
     self.database = ko.observable();
     var previousDatabase = null;

+ 1 - 0
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -942,6 +942,7 @@ ${ sqlSyntaxDropdown.sqlSyntaxDropdown() }
     <span class="execution-timer" data-bind="visible: type() != 'text' && status() != 'ready' && status() != 'loading', text: result.executionTime().toHHMMSS()" title="${ _('Execution time') }"></span>
 
     <select data-bind="selectize: computes, value: selectedCompute" class="input-medium"></select>
+    ## <div class="margin-left-10" data-bind="component: { name: 'hue-drop-down', params: { icon: 'fa-cluster', value: selectedCompute, entries: computes, labelAttribute: 'name', foreachVisible: true, searchable: true, linkTitle: '${ _ko('Active compute') }' } }" style="display: inline-block"></div>
 
     <!-- ko if: availableDatabases().length > 0 && isSqlDialect() -->
     <div class="margin-left-10" data-bind="component: { name: 'hue-drop-down', params: { icon: 'fa-database', value: database, entries: availableDatabases, foreachVisible: true, searchable: true, linkTitle: '${ _ko('Active database') }' } }" style="display: inline-block"></div>