Pārlūkot izejas kodu

HUE-6245 [dataeng] Add a basic job page in the browser

Romain Rigaux 8 gadi atpakaļ
vecāks
revīzija
cf7c42e

+ 22 - 4
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -136,8 +136,10 @@ class DataEngClusterApi(Api):
   def _api_status(self, status):
     if status in ['CREATING', 'CREATED', 'TERMINATING']:
       return 'RUNNING'
+    elif status in ['ARCHIVING']:
+      return 'SUCCEEDED'    
     else:
-      return 'FINISHED' # SUCCEEDED , KILLED and FAILED
+      return 'FAILED' # KILLED and FAILED
 
 
 class DataEngJobApi(Api):
@@ -190,16 +192,30 @@ class DataEngJobApi(Api):
     }
 
   def app(self, appid):
-    handle = DataEng(self.user).list_jobs(job_ids=[appid])
+    handle = DataEng(self.user).describe_jobs(job_ids=[appid])
+
     job = handle['jobs'][0]
 
+# {u'jobs': [{u'status': u'FAILED', u'submitterCrn': u'crn:altus:iam:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:user:/csGD5p16ZWkUateZrvVk9zm10gXQAkjqKvsIPVkQ5U=/d122f235-3a45-46a0-b3a6-e572c76a711c',
+#              u'jobId': u'9c697bc9-175d-4260-8d84-56fffc148810', u'clusterCrn': u'crn:altus:dataeng:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:cluster:praveen-hive-on-mr/2bb313d7-ffbf-42bd-a9d9-5e4f827ea799', 
+#              u'creationDate': u'2017-04-06T00:32:51.673000+00:00', u'crn': u'crn:altus:dataeng:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:job:9c697bc9-175d-4260-8d84-56fffc148810', 
+#              u'hiveJobDetails': {u'params': [], u'script': u'file:///Users/praveen/Code/Cloudera/scripts/queries/simple.hql'},
+#               u'jobType': u'HIVE', u'failureAction': u'INTERRUPT_JOB_QUEUE'}]}
+
+
     common = {
         'id': job['jobId'],
         'name': job['jobId'],
         'status': job['status'],
         'apiStatus': self._api_status(job['status']),
         'progress': 50,
-        'type': 'dataeng-job',
+        'duration': 10 * 3600,
+        'submitted': job['creationDate'],
+        'type': 'dataeng-job-%s' % job['jobType'],
+    }
+    
+    common['properties'] = {
+      'properties': job
     }
 
     return common
@@ -239,6 +255,8 @@ class DataEngJobApi(Api):
   def _api_status(self, status):
     if status in ['CREATING', 'CREATED', 'TERMINATING']:
       return 'RUNNING'
+    elif status in ['COMPLETED']:
+      return 'SUCCEEDED'
     else:
-      return 'FINISHED' # SUCCEEDED , KILLED and FAILED
+      return 'FAILED' # INTERRUPTED , KILLED and FAILED
 

+ 26 - 0
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -238,6 +238,10 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
             <!-- ko if: mainType() == 'bundles' -->
               <div class="jb-panel" data-bind="template: { name: 'bundle-page' }"></div>
             <!-- /ko -->
+            
+            <!-- ko if: mainType().startsWith('dataeng-job') -->
+              <div data-bind="template: { name: 'dataeng-job-page' }"></div>
+            <!-- /ko -->
 
           <!-- /ko -->
           <!-- /ko -->
@@ -385,6 +389,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
   <!-- /ko -->
 </script>
 
+
 <script type="text/html" id="job-yarn-page">
   <div class="row-fluid">
     <div data-bind="css:{'span2': !$root.isMini(), 'span12': $root.isMini() }">
@@ -774,6 +779,27 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 </script>
 
 
+<script type="text/html" id="dataeng-job-page">
+  <!-- ko if: type() == 'dataeng-job-HIVE' -->
+    <div data-bind="template: { name: 'job-dataeng-job-hive-page', data: $root.job() }"></div>
+  <!-- /ko -->
+</script>
+
+
+<script type="text/html" id="job-dataeng-job-hive-page">
+  ${ _('Id') } <span data-bind="text: id"></span>
+  ${ _('Name') } <span data-bind="text: name"></span>
+  ${ _('Type') } <span data-bind="text: type"></span>
+  ${ _('Status') } <span data-bind="text: status"></span>
+  ${ _('User') } <span data-bind="text: user"></span>
+  ${ _('Progress') } <span data-bind="text: progress"></span>
+  ${ _('Duration') } <span data-bind="text: duration"></span>
+  ${ _('Submitted') } <span data-bind="text: submitted"></span>
+  
+  <pre data-bind="text: ko.toJSON(properties['properties'], null, 2)"></pre>
+</script>
+
+
 <script type="text/html" id="job-actions">
   <div class="btn-group">
     <!-- ko if: hasResume -->

+ 48 - 13
desktop/libs/notebook/src/notebook/connectors/dataeng.py

@@ -138,21 +138,39 @@ class DataEng():
 
   def __init__(self, user): pass
 
-  def list_clusters(self, names=None, page_size=None, starting_token=None):
-    args = ['list-clusters']
+  def list_jobs(self, submitter_crns=None, page_size=None, starting_token=None, job_statuses=None, job_ids=None, job_types=None, creation_date_before=None,
+        creation_date_after=None, cluster_crn=None, order=None):
+    args = ['list-jobs']
 
-    if names:
-      args.extend(['--cluster-names', names])
+    if creation_date_after is None:
+      creation_date_after = (datetime.today() - timedelta(days=7)).strftime(DATE_FORMAT)
+
+    if submitter_crns:
+      args.extend(['--submitter-crns', submitter_crns])
     if page_size is not None:
       args.extend(['--page-size', str(page_size)])
     if starting_token:
       args.extend(['--starting-token', starting_token])
+    if job_statuses:
+      args.extend(['--job-statuses', job_statuses])
+    if job_ids:
+      args.extend(['--job-ids'] + job_ids)
+    if job_types:
+      args.extend(['--job-types', job_types])
+    if creation_date_before:
+      args.extend(['--creation-date-before', creation_date_before])
+    if creation_date_after:
+      args.extend(['--creation-date-after', creation_date_after])
+    if cluster_crn:
+      args.extend(['--cluster-crn', cluster_crn])
+    if order:
+      args.extend(['--order', order])
 
     return _exec(args)
 
-  def list_jobs(self, submitter_crns=None, page_size=None, starting_token=None, job_statuses=None, job_ids=None, job_types=None, creation_date_before=None,
+  def describe_jobs(self, submitter_crns=None, page_size=None, starting_token=None, job_statuses=None, job_ids=None, job_types=None, creation_date_before=None,
         creation_date_after=None, cluster_crn=None, order=None):
-    args = ['list-jobs']
+    args = ['describe-jobs']
 
     if creation_date_after is None:
       creation_date_after = (datetime.today() - timedelta(days=7)).strftime(DATE_FORMAT)
@@ -180,13 +198,6 @@ class DataEng():
 
     return _exec(args)
 
-  def create_cluster(self):
-    return _exec(['create-cluster'])
-  def delete_cluster(self):
-    return _exec(['delete-cluster'])
-  def describe_clusters(self):
-    return _exec(['describe-cluster'])
-
   def submit_hive_job(self, cluster_name, script, params=None, job_xml=None):
     args = ['submit-hive-job', '--cluster-name', cluster_name, '--script', script]
 
@@ -199,10 +210,34 @@ class DataEng():
 
   def submit_spark_job(self):
     return _exec(['submit-spark-job'])
+  
   def submit_yarn_job(self):
     return _exec(['submit-yarn-job'])
+  
   def submit_jobs(self):
     return _exec(['submit-jobs'])
 
   def terminate_jobs(self, job_ids):
     return _exec(['terminate-jobs', '--job-ids', job_ids])
+
+  
+  def list_clusters(self, names=None, page_size=None, starting_token=None):
+    args = ['list-clusters']
+
+    if names:
+      args.extend(['--cluster-names', names])
+    if page_size is not None:
+      args.extend(['--page-size', str(page_size)])
+    if starting_token:
+      args.extend(['--starting-token', starting_token])
+
+    return _exec(args)
+
+  def create_cluster(self):
+    return _exec(['create-cluster'])
+  
+  def delete_cluster(self):
+    return _exec(['delete-cluster'])
+  
+  def describe_clusters(self):
+    return _exec(['describe-cluster'])