浏览代码

HUE-8591 [cluster] Integration skeleton for Data Warehouse v2 API

Romain Rigaux 7 年之前
父节点
当前提交
4b9f986c3c

+ 2 - 0
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -52,6 +52,8 @@ def get_api(user, interface):
     return DataEngClusterApi(user)
   elif interface == 'dataware-clusters':
     return DataWarehouseClusterApi(user)
+  elif interface == 'dataware2-clusters':
+    return DataWarehouseClusterApi(user, version=2)
   elif interface == 'dataeng-jobs':
     return DataEngJobApi(user)
   elif interface == 'livy-sessions':

+ 10 - 8
apps/jobbrowser/src/jobbrowser/apis/data_warehouse.py

@@ -19,7 +19,7 @@ import logging
 
 from django.utils.translation import ugettext as _
 
-from notebook.connectors.altus import AnalyticDbApi
+from notebook.connectors.altus import AnalyticDbApi, DataWarehouse2Api
 
 from jobbrowser.apis.base_api import Api
 
@@ -32,10 +32,14 @@ RUNNING_STATES = ('QUEUED', 'RUNNING', 'SUBMITTING')
 
 class DataWarehouseClusterApi(Api):
 
-  def apps(self, filters):
-    api = AnalyticDbApi(self.user)
+  def __init__(self, user, version=1):
+    super(DataWarehouseClusterApi, self).__init__(user)
+
+    self.api = DataWarehouse2Api(self.user) if version == 2 else AnalyticDbApi(self.user) 
 
-    jobs = api.list_clusters()
+
+  def apps(self, filters):
+    jobs = self.api.list_clusters()
 
     return {
       'apps': [{
@@ -51,7 +55,7 @@ class DataWarehouseClusterApi(Api):
         'submitted': app['creationDate'],
         'canWrite': True
       } for app in sorted(jobs['clusters'], key=lambda a: a['creationDate'], reverse=True)],
-      'total': len(jobs)
+      'total': len(jobs['clusters'])
     }
 
 
@@ -63,10 +67,8 @@ class DataWarehouseClusterApi(Api):
     message = {'message': '', 'status': 0}
 
     if action.get('action') == 'kill':
-      api = AnalyticDbApi(self.user)
-
       for _id in appid:
-        result = api.delete_cluster(_id)
+        result = self.api.delete_cluster(_id)
         if result.get('error'):
           message['message'] = result.get('error')
           message['status'] = -1

+ 10 - 5
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -27,7 +27,7 @@ SUFFIX = is_mini and "-mini" or ""
 %>
 
 % if not is_embeddable:
-${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
+${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
 <%namespace name="assist" file="/assist.mako" />
 % endif
 
@@ -108,7 +108,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
             <li class="app-header">
               <a href="/${app_name}">
                 <img src="${ static('jobbrowser/art/icon_jobbrowser_48.png') }" class="app-icon" alt="${ _('Job browser icon') }"/>
-                ${ _('Job Browser') }
+                ${ _('Data Warehouse') }
               </a>
             </li>
             <!-- ko foreach: availableInterfaces -->
@@ -2486,7 +2486,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       self.selectedJobs = ko.observableArray();
 
       self.hasKill = ko.pureComputed(function() {
-        return ['jobs', 'workflows', 'schedules', 'bundles', 'queries', 'dataeng-jobs', 'dataeng-clusters', 'dataware-clusters'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
+        return ['jobs', 'workflows', 'schedules', 'bundles', 'queries', 'dataeng-jobs', 'dataeng-clusters', 'dataware-clusters', 'dataware2-clusters'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
       });
       self.killEnabled = ko.pureComputed(function() {
         return self.hasKill() && self.selectedJobs().length > 0 && $.grep(self.selectedJobs(), function(job) {
@@ -2789,10 +2789,13 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('yarn') != -1 && self.clusterType() != '${ ANALYTIC_DB }' && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
         };
         var dataEngInterfaceCondition = function () {
-          return self.cluster() && self.cluster()['type'].indexOf('altus-de') >= 0;
+          return self.cluster() && self.cluster()['type'] == 'altus-de';
         };
         var dataWarehouseInterfaceCondition = function () {
-          return self.cluster() && self.cluster()['type'].indexOf('altus-dw') >= 0;
+          return self.cluster() && self.cluster()['type'] == 'altus-dw';
+        };
+        var dataWarehouse2InterfaceCondition = function () {
+          return self.cluster() && self.cluster()['type'] == 'altus-dw2';
         };
         var schedulerInterfaceCondition = function () {
           return '${ user.has_hue_permission(action="access", app="oozie") }' == 'True' && self.clusterType() != '${ ANALYTIC_DB }' && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
@@ -2812,6 +2815,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           {'interface': 'dataeng-jobs', 'label': '${ _ko('Jobs') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'dataeng-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'dataware-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataWarehouseInterfaceCondition},
+          {'interface': 'dataware2-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataWarehouse2InterfaceCondition},
           {'interface': 'queries', 'label': '${ _ko('Queries') }', 'condition': queryInterfaceCondition},
           {'interface': 'workflows', 'label': '${ _ko('Workflows') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'schedules', 'label': '${ _ko('Schedules') }', 'condition': schedulerInterfaceCondition},
@@ -2996,6 +3000,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           case 'bundles':
           case 'dataeng-clusters':
           case 'dataware-clusters':
+          case 'dataware2-clusters':
           case 'dataeng-jobs':
           case 'livy-sessions':
             self.selectInterface(h);

+ 1 - 1
desktop/core/src/desktop/templates/assist.mako

@@ -929,7 +929,7 @@ from desktop.views import _ko
   <script type="text/html" id="assist-namespaces-template">
     <div class="assist-flex-header">
       <div class="assist-inner-header">
-        ${_('Namespaces')}
+        ${_('Clusters')}
         <!-- ko template: 'assist-namespace-header-actions' --><!-- /ko -->
       </div>
     </div>

+ 6 - 21
desktop/core/src/desktop/templates/ko_components/ko_multi_cluster_sidebar.mako

@@ -168,22 +168,6 @@ from desktop.views import _ko
             label: '${ _('Catalog') }',
             url: '/',
             icon: 'altus-icon altus-sdx'
-          },{
-            label: '${ _('Streaming') }',
-            url: '/',
-            icon: 'fa fa-sitemap'
-          },{
-            label: '${ _('Operational DB') }',
-            url: '/',
-            icon: 'altus-icon altus-adb',
-            items: [{
-                label: '${ _('Kudu') }',
-                url: '/'
-              },{
-                label: '${ _('HBase') }',
-                url: '/'
-              }
-            ]
           }
         ]
       }, {
@@ -191,7 +175,7 @@ from desktop.views import _ko
         items: [{
             label: '${ _('Data Warehouse') }',
             icon: 'altus-icon altus-dashboard',
-            url: '/'
+            url: '/hue/jobbrowser'
           }, {
             label: '${ _('Data Engineering') }',
             icon: 'altus-icon altus-de',
@@ -203,11 +187,12 @@ from desktop.views import _ko
                 url: '/'
               }
             ]
-          }, {
-            label: '${ _('Scheduling') }',
-            icon: 'altus-icon altus-workload',
-            url: '/'
           }
+          ##, {
+          ##  label: '${ _('Scheduling') }',
+          ##  icon: 'altus-icon altus-workload',
+          ##  url: '/'
+          ##}
         ]
       }
         ##  , {

+ 16 - 3
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -27,6 +27,8 @@ from navoptapi.api_lib import ApiLib
 
 
 from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.rest.http_client import HttpClient
+from desktop.lib.rest.resource import Resource
 
 
 LOG = logging.getLogger(__name__)
@@ -268,9 +270,15 @@ class AnalyticDbApi():
     return _exec('dataware', 'deleteCluster', {'clusterName': cluster_id})
 
 
-class DataWarehouseAutoScaleApi():
+class DataWarehouse2Api():
 
-  def __init__(self, user): pass
+  def __init__(self, user=None):
+    self._api_url = '/warehouse/api'
+
+    self.user = user
+    self._client = HttpClient(self._api_url, logger=LOG)
+    self._client.set_verify(False)
+    self._root = Resource(self._client)
 
   def create_cluster(self, cluster_name, cdh_version, cpu_minimum, cpu_maximum, memory_minimum, memory_maximum):
     params = {
@@ -285,7 +293,12 @@ class DataWarehouseAutoScaleApi():
     return _exec('dw', 'createCluster', params)
 
   def list_clusters(self):
-    return _exec('dw', 'listClusters')
+    try:
+      return self._root.post('listClusters', contenttype="application/json")
+    except:
+      return {'clusters': [
+        {'crn': 'crn1', 'clusterName': 'clusterName1', 'status': 'CREATED', 'workersGroupSize': 1, 'instanceType': 'Medium', 'cdhVersion': '6.0', 'creationDate': 'September 14, 2018 12:48 PM'}
+      ]}
 
   def delete_cluster(self, cluster_id):
     return _exec('dw', 'deleteCluster', {'clusterName': cluster_id})