Эх сурвалжийг харах

HUE-6908 [jb] Skeleton of Spark API

Romain Rigaux 8 жил өмнө
parent
commit
9bff785

+ 144 - 0
apps/jobbrowser/src/jobbrowser/apis/livy_api.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from datetime import datetime,  timedelta
+
+from django.utils.translation import ugettext as _
+
+from spark.job_server_api import get_api
+
+from jobbrowser.apis.base_api import Api
+
+
+LOG = logging.getLogger(__name__)
+
+
+class LivySessionsApi(Api):
+
+  def apps(self, filters):
+    api = get_api(self.user)
+
+    jobs = api.list_clusters()
+
+    return {
+      'apps': [{
+        'id': app['crn'],
+        'name': '%(clusterName)s' % app,
+        'status': app['status'],
+        'apiStatus': self._api_status(app['status']),
+        'type': '%(serviceType)s %(workersGroupSize)s %(instanceType)s %(cdhVersion)s' % app,
+        'user': app['clusterName'].split('-', 1)[0],
+        'progress': 100,
+        'queue': 'group',
+        'duration': 1,
+        'submitted': app['creationDate']
+      } for app in jobs['clusters']],
+      'total': len(jobs)
+    }
+
+
+  def app(self, appid):
+    return {}
+
+
+  def action(self, appid, action):
+    return {}
+
+
+  def logs(self, appid, app_type, log_name=None):
+    return {'logs': ''}
+
+
+  def profile(self, appid, app_type, app_property):
+    return {}
+
+  def _api_status(self, status):
+    if status in ['CREATING', 'CREATED', 'TERMINATING']:
+      return 'RUNNING'
+    elif status in ['ARCHIVING', 'COMPLETED']:
+      return 'SUCCEEDED'
+    else:
+      return 'FAILED' # KILLED and FAILED
+
+
+class LivyJobApi(Api):
+
+  def apps(self, filters):
+    kwargs = {}
+
+    api = get_api(self.user)
+
+    jobs = api.list_jobs(**kwargs)
+
+    return {
+      'apps': [{
+        'id': app['jobId'],
+        'name': app['creationDate'],
+        'status': app['status'],
+        'apiStatus': self._api_status(app['status']),
+        'type': app['jobType'],
+        'user': '',
+        'progress': 100,
+        'duration': 10 * 3600,
+        'submitted': app['creationDate']
+      } for app in jobs['jobs']],
+      'total': len(jobs)
+    }
+
+  def app(self, appid):
+    handle = DataEng(self.user).describe_job(job_id=appid)
+
+    job = handle['job']
+
+    common = {
+        'id': job['jobId'],
+        'name': job['jobId'],
+        'status': job['status'],
+        'apiStatus': self._api_status(job['status']),
+        'progress': 50,
+        'duration': 10 * 3600,
+        'submitted': job['creationDate'],
+        'type': 'dataeng-job-%s' % job['jobType'],
+    }
+
+    common['properties'] = {
+      'properties': job
+    }
+
+    return common
+
+
+  def action(self, appid, action):
+    return {}
+
+
+  def logs(self, appid, app_type, log_name=None):
+    return {'logs': ''}
+
+
+  def profile(self, appid, app_type, app_property):
+    return {}
+
+  def _api_status(self, status):
+    if status in ['CREATING', 'CREATED', 'TERMINATING']:
+      return 'RUNNING'
+    elif status in ['COMPLETED']:
+      return 'SUCCEEDED'
+    else:
+      return 'FAILED' # INTERRUPTED , KILLED, TERMINATED and FAILED