Преглед изворни кода

HUE-6245 [dataeng] Add list of jobs to Job Browser

Romain Rigaux пре 8 година
родитељ
комит
3fdb067

+ 34 - 22
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -18,11 +18,13 @@
 import logging
 import json
 
+from datetime import datetime,  timedelta
+
 from django.utils.translation import ugettext as _
 
 from jobbrowser.apis.base_api import Api, MockDjangoRequest, _extract_query_params
 from liboozie.oozie_api import get_oozie
-from notebook.connectors.dataeng_batch import DataEng
+from notebook.connectors.dataeng_batch import DataEng, DATE_FORMAT
 
 
 LOG = logging.getLogger(__name__)
@@ -56,7 +58,8 @@ class DataEngClusterApi(Api):
 
     jobs = api.list_clusters()
     
-    return [{
+    return {
+      'apps': [{
         'id': app['clusterName'],
         'name': '%(workersGroupSize)s %(instanceType)s %(cdhVersion)s' % app,
         'status': app['status'],
@@ -66,7 +69,10 @@ class DataEngClusterApi(Api):
         'progress': 100,
         'duration': 10 * 3600,
         'submitted': app['creationDate']
-    } for app in jobs['clusters']]
+      } for app in jobs['clusters']],
+      'total': None
+    }
+            
 
 
   def app(self, appid):
@@ -137,7 +143,7 @@ class DataEngClusterApi(Api):
 class DataEngJobApi(Api):
 
   def apps(self, filters):
-#     kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
+    kwargs = {}
 # 
 #     text_filters = _extract_query_params(filters)
 # 
@@ -146,9 +152,15 @@ class DataEngJobApi(Api):
 #     elif 'user' in text_filters:
 #       kwargs['filters'].append(('user', text_filters['username']))
 # 
-#     if 'time' in filters:
-#       kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])
-# 
+    if 'time' in filters:
+      if filters['time']['time_unit'] == 'minutes':
+        delta = timedelta(minutes=int(filters['time']['time_value']))
+      elif filters['time']['time_unit'] == 'hours':
+        delta = timedelta(hours=int(filters['time']['time_value']))
+      else:
+        delta = timedelta(days=int(filters['time']['time_value']))                                                                    
+      kwargs['creation_date_after'] = (datetime.today() - delta).strftime(DATE_FORMAT)
+
 #     if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
 #       kwargs['filters'].extend([('text', text_filters.get('text'))])
 # 
@@ -160,20 +172,22 @@ class DataEngJobApi(Api):
 
     api = DataEng(self.user)
 
-    jobs = api.list_jobs()
+    jobs = api.list_jobs(**kwargs)
 
-    return [{
-        'id': app.id,
-        'name': app.appName,
-        'status': app.status,
-        'apiStatus': self._api_status(app.status),
-        'type': 'workflow',
-        'user': app.user,
-        'progress': app.get_progress(),
+    return {
+      'apps': [{
+        'id': app['jobId'],
+        'name': app['creationDate'],
+        'status': app['status'],
+        'apiStatus': self._api_status(app['status']),
+        'type': app['jobType'],
+        'user': '',
+        'progress': 100,
         'duration': 10 * 3600,
-        'submitted': 10 * 3600
-    } for app in wf_list.jobs]
-
+        'submitted': app['creationDate']
+      } for app in jobs['jobs']],
+      'total': None
+    }
 
   def app(self, appid):
     oozie_api = get_oozie(self.user)
@@ -234,10 +248,8 @@ class DataEngJobApi(Api):
     return {}
 
   def _api_status(self, status):
-    if status in ['PREP', 'RUNNING']:
+    if status in ['CREATING', 'CREATED', 'TERMINATING']:
       return 'RUNNING'
-    elif status == 'SUSPENDED':
-      return 'PAUSED'
     else:
       return 'FINISHED' # SUCCEEDED , KILLED and FAILED
 

+ 26 - 19
desktop/libs/notebook/src/notebook/connectors/dataeng_batch.py

@@ -21,6 +21,8 @@ import re
 import subprocess
 import time
 
+from datetime import datetime,  timedelta
+
 from django.core.urlresolvers import reverse
 from django.http import QueryDict
 from django.utils.translation import ugettext as _
@@ -50,6 +52,8 @@ def _exec(args):
 
   return response
 
+DATE_FORMAT = "%Y-%m-%d"
+
 
 class DataEngBatchApi(Api):
 
@@ -219,42 +223,45 @@ class DataEng():
 
   def __init__(self, user): pass
 
-  def list_clusters(self, names=None, size=None, starting_token=None):
+  def list_clusters(self, names=None, page_size=None, starting_token=None):
     args = ['list-clusters']
 
     if names:
-      args.append('--cluster-names')
-    if size is not None:
-      args.append('--page-size')
+      args.extend(['--cluster-names', names])
+    if page_size is not None:
+      args.extend(['--page-size', str(page_size)])
     if starting_token:
-      args.append('--starting-token')
+      args.extend(['--starting-token', starting_token])
 
     return _exec(args)
 
-  def list_jobs(self, submitter_crns=None, size=None, starting_token=None, job_statuses=None, job_ids=None, job_types=None, creation_date_before=None,
-        creatation_date_after=None, cluster_crn=None, order=None):
+  def list_jobs(self, submitter_crns=None, page_size=None, starting_token=None, job_statuses=None, job_ids=None, job_types=None, creation_date_before=None,
+        creation_date_after=None, cluster_crn=None, order=None):
     args = ['list-jobs']
 
+    if creation_date_after is None:
+      creation_date_after = (datetime.today() - timedelta(days=7)).strftime(DATE_FORMAT)
+
     if submitter_crns:
-      args.append('--submitter-crns')
-    if size is not None:
-      args.append('--page-size')
+      args.extend(['--submitter-crns', submitter_crns])
+    if page_size is not None:
+      args.extend(['--page-size', str(page_size)])
     if starting_token:
-      args.append('--starting-token')
+      args.extend(['--starting-token', starting_token])
     if job_statuses:
-      args.append('--job-statuses')
+      args.extend(['--job-statuses', job_statuses])
     if job_ids:
-      args.append('--job-ids')
+      args.extend(['--job-ids', job_ids])
     if job_types:
-      args.append('--job-types')
+      args.extend(['--job-types', job_types])
     if creation_date_before:
-      args.append('--creation-date-before')
-    if creatation_date_after:
-      args.append('--creation-date-after')
+      args.extend(['--creation-date-before', creation_date_before])
+    if creation_date_after:
+      args.extend(['--creation-date-after', creation_date_after])
     if cluster_crn:
-      args.append('--cluster-crn')
+      args.extend(['--cluster-crn', cluster_crn])
     if order:
-      args.append('--order')
+      args.extend(['--order', order])
 
     return _exec(args)