data_warehouse.py 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import logging
  18. from datetime import datetime
  19. from dateutil import parser
  20. from django.utils import timezone
  21. from django.utils.translation import ugettext as _
  22. from notebook.connectors.altus import AnalyticDbApi, DataWarehouse2Api
  23. from jobbrowser.apis.base_api import Api
  24. LOG = logging.getLogger(__name__)
  25. RUNNING_STATES = ('QUEUED', 'RUNNING', 'SUBMITTING')
  26. class DataWarehouseClusterApi(Api):
  27. def __init__(self, user, version=1):
  28. super(DataWarehouseClusterApi, self).__init__(user)
  29. self.version = version
  30. self.api = DataWarehouse2Api(self.user) if version == 2 else AnalyticDbApi(self.user)
  31. def apps(self, filters):
  32. jobs = self.api.list_clusters()
  33. return {
  34. 'apps': [{
  35. 'id': app['crn'],
  36. 'name': '%(clusterName)s' % app,
  37. 'status': app['status'],
  38. 'apiStatus': self._api_status(app['status']),
  39. 'type': 'Altus %(workersGroupSize)s %(instanceType)s %(cdhVersion)s' % app,
  40. 'user': app['clusterName'].split('-', 1)[0],
  41. 'progress': app.get('progress', 100),
  42. 'queue': 'group',
  43. 'duration': (datetime.now() - parser.parse(app['creationDate']).replace(tzinfo=None)).seconds * 1000,
  44. 'submitted': app['creationDate'],
  45. 'canWrite': True
  46. } for app in sorted(jobs['clusters'], key=lambda a: a['creationDate'], reverse=True)],
  47. 'total': len(jobs['clusters'])
  48. }
  49. def app(self, appid):
  50. handle = self.api.describe_cluster(cluster_id=appid)
  51. cluster = handle['cluster']
  52. common = {
  53. 'id': cluster['crn'],
  54. 'name': cluster['clusterName'],
  55. 'status': cluster['status'],
  56. 'apiStatus': self._api_status(cluster['status']),
  57. 'progress': 50 if self._api_status(cluster['status']) == 'RUNNING' else 100,
  58. 'duration': 10 * 3600,
  59. 'submitted': cluster['creationDate'],
  60. 'type': 'dataware2-cluster' if self.version == 2 else 'dataware-cluster',
  61. 'canWrite': True
  62. }
  63. common['properties'] = {
  64. 'properties': cluster
  65. }
  66. return common
  67. def action(self, appid, action):
  68. message = {'message': '', 'status': 0}
  69. if action.get('action') == 'kill':
  70. for _id in appid:
  71. result = self.api.delete_cluster(_id)
  72. if result.get('error'):
  73. message['message'] = result.get('error')
  74. message['status'] = -1
  75. elif result.get('contents') and message.get('status') != -1:
  76. message['message'] = result.get('contents')
  77. return message;
  78. def logs(self, appid, app_type, log_name=None, is_embeddable=False):
  79. return {'logs': ''}
  80. def profile(self, appid, app_type, app_property):
  81. return {}
  82. def _api_status(self, status):
  83. if status in ['CREATING', 'CREATED', 'ONLINE', 'SCALING_UP', 'SCALING_DOWN', 'STOPPED', 'STARTING']:
  84. return 'RUNNING'
  85. elif status in ['ARCHIVING', 'COMPLETED', 'TERMINATING']:
  86. return 'SUCCEEDED'
  87. else:
  88. return 'FAILED' # KILLED and FAILED