Эх сурвалжийг харах

HUE-8555 [jb] List data warehouse clusters

Romain Rigaux 7 жил өмнө
parent
commit
f0d99d2628

+ 3 - 0
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -31,6 +31,7 @@ LOG = logging.getLogger(__name__)
 def get_api(user, interface):
 def get_api(user, interface):
   from jobbrowser.apis.bundle_api import BundleApi
   from jobbrowser.apis.bundle_api import BundleApi
   from jobbrowser.apis.data_eng_api import DataEngClusterApi, DataEngJobApi
   from jobbrowser.apis.data_eng_api import DataEngClusterApi, DataEngJobApi
+  from jobbrowser.apis.data_warehouse import DataWarehouseClusterApi
   from jobbrowser.apis.livy_api import LivySessionsApi, LivyJobApi
   from jobbrowser.apis.livy_api import LivySessionsApi, LivyJobApi
   from jobbrowser.apis.job_api import JobApi
   from jobbrowser.apis.job_api import JobApi
   from jobbrowser.apis.query_api import QueryApi
   from jobbrowser.apis.query_api import QueryApi
@@ -49,6 +50,8 @@ def get_api(user, interface):
     return BundleApi(user)
     return BundleApi(user)
   elif interface == 'dataeng-clusters':
   elif interface == 'dataeng-clusters':
     return DataEngClusterApi(user)
     return DataEngClusterApi(user)
+  elif interface == 'dataware-clusters':
+    return DataWarehouseClusterApi(user)
   elif interface == 'dataeng-jobs':
   elif interface == 'dataeng-jobs':
     return DataEngJobApi(user)
     return DataEngJobApi(user)
   elif interface == 'livy-sessions':
   elif interface == 'livy-sessions':

+ 92 - 0
apps/jobbrowser/src/jobbrowser/apis/data_warehouse.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from django.utils.translation import ugettext as _
+
+from notebook.connectors.altus import AnalyticDbApi
+
+from jobbrowser.apis.base_api import Api
+
+
+LOG = logging.getLogger(__name__)
+
+
+RUNNING_STATES = ('QUEUED', 'RUNNING', 'SUBMITTING')
+
+
+class DataWarehouseClusterApi(Api):
+
+  def apps(self, filters):
+    api = AnalyticDbApi(self.user)
+
+    jobs = api.list_clusters()
+
+    return {
+      'apps': [{
+        'id': app['crn'],
+        'name': '%(clusterName)s' % app,
+        'status': app['status'],
+        'apiStatus': self._api_status(app['status']),
+        'type': 'Altus %(workersGroupSize)s %(instanceType)s %(cdhVersion)s' % app,
+        'user': app['clusterName'].split('-', 1)[0],
+        'progress': 100,
+        'queue': 'group',
+        'duration': 1,
+        'submitted': app['creationDate'],
+        'canWrite': True
+      } for app in jobs['clusters']],
+      'total': len(jobs)
+    }
+
+
+  def app(self, appid):
+    return {}
+
+
+  def action(self, appid, action):
+    message = {'message': '', 'status': 0}
+
+    if action.get('action') == 'kill':
+      api = AnalyticDbApi(self.user)
+
+      for _id in appid:
+        result = api.delete_cluster(_id)
+        if result.get('error'):
+          message['message'] = result.get('error')
+          message['status'] = -1
+        elif result.get('contents') and message.get('status') != -1:
+          message['message'] = result.get('contents')
+
+    return message;
+
+
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
+    return {'logs': ''}
+
+
+  def profile(self, appid, app_type, app_property):
+    return {}
+
+  def _api_status(self, status):
+    if status in ['CREATING', 'CREATED']:
+      return 'RUNNING'
+    elif status in ['ARCHIVING', 'COMPLETED', 'TERMINATING']:
+      return 'SUCCEEDED'
+    else:
+      return 'FAILED' # KILLED and FAILED

+ 8 - 4
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -2468,7 +2468,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       self.selectedJobs = ko.observableArray();
       self.selectedJobs = ko.observableArray();
 
 
       self.hasKill = ko.pureComputed(function() {
       self.hasKill = ko.pureComputed(function() {
-        return ['jobs', 'workflows', 'schedules', 'bundles', 'queries', 'dataeng-jobs', 'dataeng-clusters'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
+        return ['jobs', 'workflows', 'schedules', 'bundles', 'queries', 'dataeng-jobs', 'dataeng-clusters', 'dataware-clusters'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
       });
       });
       self.killEnabled = ko.pureComputed(function() {
       self.killEnabled = ko.pureComputed(function() {
         return self.hasKill() && self.selectedJobs().length > 0 && $.grep(self.selectedJobs(), function(job) {
         return self.hasKill() && self.selectedJobs().length > 0 && $.grep(self.selectedJobs(), function(job) {
@@ -2771,9 +2771,11 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('yarn') != -1 && self.clusterType() != '${ ANALYTIC_DB }' && (!self.compute() || self.compute()['type'].indexOf('altus') == -1);
           return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('yarn') != -1 && self.clusterType() != '${ ANALYTIC_DB }' && (!self.compute() || self.compute()['type'].indexOf('altus') == -1);
         };
         };
         var dataEngInterfaceCondition = function () {
         var dataEngInterfaceCondition = function () {
-          // return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('dataeng') != -1;
-          return self.compute() && self.compute()['type'].indexOf('altus') >= 0;
-        };
+          return self.compute() && self.compute()['type'].indexOf('altus-de') >= 0;
+        }
+        var dataWarehouseInterfaceCondition = function () {
+          return self.compute() && self.compute()['type'].indexOf('altus-dw') >= 0;
+        }
         var schedulerInterfaceCondition = function () {
         var schedulerInterfaceCondition = function () {
           return '${ user.has_hue_permission(action="access", app="oozie") }' == 'True' && self.clusterType() != '${ ANALYTIC_DB }' && (!self.compute() || self.compute()['type'].indexOf('altus') == -1);
           return '${ user.has_hue_permission(action="access", app="oozie") }' == 'True' && self.clusterType() != '${ ANALYTIC_DB }' && (!self.compute() || self.compute()['type'].indexOf('altus') == -1);
         };
         };
@@ -2788,6 +2790,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           {'interface': 'jobs', 'label': '${ _ko('Jobs') }', 'condition': jobsInterfaceCondition},
           {'interface': 'jobs', 'label': '${ _ko('Jobs') }', 'condition': jobsInterfaceCondition},
           {'interface': 'dataeng-jobs', 'label': '${ _ko('Jobs') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'dataeng-jobs', 'label': '${ _ko('Jobs') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'dataeng-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'dataeng-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataEngInterfaceCondition},
+          {'interface': 'dataware-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataWarehouseInterfaceCondition},
           {'interface': 'queries', 'label': '${ _ko('Queries') }', 'condition': queryInterfaceCondition},
           {'interface': 'queries', 'label': '${ _ko('Queries') }', 'condition': queryInterfaceCondition},
           {'interface': 'workflows', 'label': '${ _ko('Workflows') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'workflows', 'label': '${ _ko('Workflows') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'schedules', 'label': '${ _ko('Schedules') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'schedules', 'label': '${ _ko('Schedules') }', 'condition': schedulerInterfaceCondition},
@@ -2971,6 +2974,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           case 'schedules':
           case 'schedules':
           case 'bundles':
           case 'bundles':
           case 'dataeng-clusters':
           case 'dataeng-clusters':
+          case 'dataware-clusters':
           case 'dataeng-jobs':
           case 'dataeng-jobs':
           case 'livy-sessions':
           case 'livy-sessions':
             self.selectInterface(h);
             self.selectInterface(h);

+ 25 - 3
desktop/core/src/desktop/api2.py

@@ -131,7 +131,7 @@ def get_context_computes(request, interface):
 
 
   clusters = get_clusters(request.user).values()
   clusters = get_clusters(request.user).values()
 
 
-  if interface == 'hive' or interface == 'impala' or interface == 'oozie' or interface == 'jobs' or interface == 'report':
+  if interface == 'hive' or interface == 'impala' or interface == 'oozie' or interface == 'report':
     computes.extend([{
     computes.extend([{
         'id': cluster['id'],
         'id': cluster['id'],
         'name': cluster['name'],
         'name': cluster['name'],
@@ -141,7 +141,7 @@ def get_context_computes(request, interface):
       } for cluster in clusters if cluster.get('type') == 'direct'
       } for cluster in clusters if cluster.get('type') == 'direct'
     ])
     ])
 
 
-  if interface == 'impala' or interface == 'jobs' or interface == 'report':
+  if interface == 'impala' or interface == 'report':
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
       computes.extend([{
       computes.extend([{
           'id': cluster.get('crn'),
           'id': cluster.get('crn'),
@@ -152,7 +152,7 @@ def get_context_computes(request, interface):
         } for cluster in AnalyticDbApi(request.user).list_clusters()['clusters'] if cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515']
         } for cluster in AnalyticDbApi(request.user).list_clusters()['clusters'] if cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515']
       )
       )
 
 
-  if interface == 'oozie' or interface == 'jobs' or interface == 'spark2':
+  if interface == 'oozie' or interface == 'spark2':
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
       computes.extend([{
       computes.extend([{
           'id': cluster.get('crn'),
           'id': cluster.get('crn'),
@@ -166,6 +166,28 @@ def get_context_computes(request, interface):
       )
       )
       # TODO if interface == 'spark2' keep only SPARK type
       # TODO if interface == 'spark2' keep only SPARK type
 
 
+  if interface == 'jobs':
+    for cluster in clusters:
+      cluster = {
+        'id': cluster.get('id'),
+        'name': cluster.get('name'),
+        'status': 'CREATED',
+        'environmentType': cluster.get('type'),
+        'serviceType': cluster.get('interface'),
+        'namespace': '',
+        'type': cluster.get('type')
+      }
+        
+      if cluster.get('type') == 'altus':
+        cluster['name'] = 'Altus DE'
+        cluster['type'] = 'altus-de'
+        computes.append(cluster)
+        cluster = cluster.copy()
+        cluster['name'] = 'Altus Data Warehouse'
+        cluster['type'] = 'altus-dw'
+
+      computes.append(cluster)
+
   response[interface] = computes
   response[interface] = computes
   response['status'] = 0
   response['status'] = 0
 
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -38,7 +38,7 @@ def _exec(service, command, parameters=None):
   if parameters is None:
   if parameters is None:
     parameters = {}
     parameters = {}
 
 
-  if service == 'analyticdb':
+  if service == 'dataware':
     hostname = ALTUS.HOSTNAME_ANALYTICDB.get()
     hostname = ALTUS.HOSTNAME_ANALYTICDB.get()
   elif service == 'dataeng':
   elif service == 'dataeng':
     hostname = ALTUS.HOSTNAME_DATAENG.get()
     hostname = ALTUS.HOSTNAME_DATAENG.get()