Browse Source

HUE-2691 [jobbrowser] Fix job browser rendering non-MapReduce/Spark jobs

This also adds displaying the app type, and using the job name if
it's available for old MAPREDUCE jobs.
Erick Tryzelaar 10 years ago
parent
commit
cbdad59348

+ 113 - 2
apps/jobbrowser/src/jobbrowser/templates/job.mako

@@ -108,6 +108,8 @@ ${ comps.menubar() }
         <ul class="nav nav-list">
           <li class="nav-header">${_('App ID')}</li>
           <li class="white hellipsify">${job.jobId_short}</li>
+          <li class="nav-header">${_('Type')}</li>
+          <li class="white hellipsify">${job.applicationType}</li>
           <li class="nav-header">${_('User')}</li>
           <li class="white">${job.user}</li>
           <li class="nav-header">${_('Status')}</li>
@@ -202,7 +204,8 @@ ${ comps.menubar() }
   </div>
 </div>
 
-% else:
+% elif job.applicationType == 'MR2':
+
 <div class="container-fluid">
   <div class="row-fluid">
     <div class="span2">
@@ -210,6 +213,8 @@ ${ comps.menubar() }
         <ul class="nav nav-list">
           <li class="nav-header">${_('Job ID')}</li>
           <li class="white hellipsify">${job.jobId_short}</li>
+          <li class="nav-header">${_('Type')}</li>
+          <li class="white hellipsify">${job.applicationType or 'MR2'}</li>
           <li class="nav-header">${_('User')}</li>
           <li class="white">${job.user}</li>
           <li class="nav-header">${_('Status')}</li>
@@ -249,7 +254,11 @@ ${ comps.menubar() }
     </div>
     <div class="span10">
       <div class="card card-small">
-	<h1 class="card-heading simple">${_('Job: %(jobId)s') % dict(jobId=job.jobId_short)}</h1>
+          % if hasattr(job, 'name'):
+            <h1 class="card-heading simple">${_(job.name)}</h1>
+          % else:
+	          <h1 class="card-heading simple">${_('Job: %(jobId)s') % dict(jobId=job.jobId_short)}</h1>
+          % endif
           <div class="card-body">
             <p>
 
@@ -411,6 +420,108 @@ ${ comps.menubar() }
     </div>
   </div>
 </div>
+
+% else:
+
+<div class="container-fluid">
+  <div class="row-fluid">
+    <div class="span2">
+      <div class="sidebar-nav" style="padding-top: 0">
+        <ul class="nav nav-list">
+          <li class="nav-header">${_('App ID')}</li>
+          <li class="white hellipsify">${job.jobId_short}</li>
+          <li class="nav-header">${_('Type')}</li>
+          <li class="white hellipsify">${job.applicationType}</li>
+          <li class="nav-header">${_('User')}</li>
+          <li class="white">${job.user}</li>
+          <li class="nav-header">${_('Status')}</li>
+          <li class="white" id="jobStatus">&nbsp;</li>
+
+          % if job.trackingUrl:
+            <li class="nav-header">${_('Tracking URL')}</li>
+            <li><a href="${job.trackingUrl }" target="_blank"><i class="fa fa-tasks"></i> ${_('Tracking URL')}</a></li>
+          % endif
+
+          <li class="nav-header">${_('Progress')}</li>
+          <li class="white">${job.progress}%</li>
+          <li class="nav-header">${_('Duration')}</li>
+          <li class="white">${job.durationFormatted}</li>
+          <li class="nav-header killJob">${_('Actions')}</li>
+          <li id="killJobContainer" class="white killJob"></li>
+        </ul>
+      </div>
+    </div>
+    <div class="span10">
+      <div class="card card-small">
+        <h1 class="card-heading simple">${_(job.name)}</h1>
+        <div class="card-body">
+          <ul class="nav nav-tabs">
+            <li class="active"><a href="#metadata" data-toggle="tab">${_('Metadata')}</a></li>
+          </ul>
+          <div class="tab-content">
+            <div class="tab-pane active" id="metadata">
+              <table class="table table-condensed">
+                <thead>
+                  <th>${_('Name')}</th>
+                  <th>${_('Value')}</th>
+                </thead>
+                <tbody>
+
+                  % if job.trackingUrl:
+
+                  <tr>
+                    <td>${_('Jobs')}</td>
+                    <td><a href="${job.trackingUrl}">${job.trackingUrl}</a></td>
+                  </tr>
+
+                  % endif
+
+                  % if hasattr(job, 'amHostHttpAddress'):
+
+                  <tr>
+                    <td>${_('Host')}</td>
+                    <td><a href="http://${job.amHostHttpAddress}">http://${job.amHostHttpAddress}</a></td>
+                  </tr>
+
+                  %endif
+
+                  <tr>
+                    <td>${_('Queue Name')}</td>
+                    <td>${job.queueName}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Started')}</td>
+                    <td>${job.startTimeFormatted}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Finished')}</td>
+                    <td>${job.finishTimeFormatted}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Pre-empted Resource VCores')}</td>
+                    <td>${job.preemptedResourceVCores}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('VCore seconds')}</td>
+                    <td>${job.vcoreSeconds}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Memory seconds')}</td>
+                    <td>${job.memorySeconds}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Diagnostics')}</td>
+                    <td>${job.diagnostics}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
 % endif
 
 

+ 107 - 16
apps/jobbrowser/src/jobbrowser/tests.py

@@ -415,7 +415,7 @@ class TestMapReduce2NoHadoop:
   def test_jobs(self):
     response = self.c.get('/jobbrowser/?format=json')
     response_content = json.loads(response.content)
-    assert_equal(len(response_content['jobs']), 2)
+    assert_equal(len(response_content['jobs']), 4)
 
     response = self.c.get('/jobbrowser/jobs/?format=json&text=W=MapReduce-copy2')
     response_content = json.loads(response.content)
@@ -437,6 +437,14 @@ class TestMapReduce2NoHadoop:
     response = self.c.get('/jobbrowser/jobs/job_1356251510842_0009')
     assert_equal(response.context['job'].jobId, 'job_1356251510842_0009')
 
+  def test_spark_job(self):
+    response = self.c.get('/jobbrowser/jobs/application_1428442704693_0006')
+    assert_equal(response.context['job'].jobId, 'application_1428442704693_0006')
+
+  def test_yarn_job(self):
+    response = self.c.get('/jobbrowser/jobs/application_1428442704693_0007')
+    assert_equal(response.context['job'].jobId, 'application_1428442704693_0007')
+
   def job_not_assigned(self):
     response = self.c.get('/jobbrowser/jobs/job_1356251510842_0009/job_not_assigned//my_url')
     assert_equal(response.context['jobid'], 'job_1356251510842_0009')
@@ -469,23 +477,102 @@ class TestMapReduce2NoHadoop:
 class MockResourceManagerApi:
   APPS = {
     'application_1356251510842_0054': {
-        u'finishedTime': 1356961070119, u'name': u'oozie:launcher:T=map-reduce:W=MapReduce-copy:A=Sleep:ID=0000004-121223003201296-oozie-oozi-W',
-        u'amContainerLogs': u'http://runreal:8042/node/containerlogs/container_1356251510842_0054_01_000001/romain', u'clusterId': 1356251510842,
-        u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0054/jobhistory/job/job_1356251510842_0054', u'amHostHttpAddress': u'runreal:8042',
-        u'startedTime': 1356961057225, u'queue': u'default', u'state': u'RUNNING', u'elapsedTime': 12894, u'finalStatus': u'UNDEFINED', u'diagnostics': u'',
-        u'progress': 100.0, u'trackingUI': u'History', u'id': u'application_1356251510842_0054', u'user': u'test',
+        u'finishedTime': 1356961070119,
+        u'name': u'oozie:launcher:T=map-reduce:W=MapReduce-copy:A=Sleep:ID=0000004-121223003201296-oozie-oozi-W',
+        u'amContainerLogs': u'http://localhost:8042/node/containerlogs/container_1356251510842_0054_01_000001/romain',
+        u'clusterId': 1356251510842,
+        u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0054/jobhistory/job/job_1356251510842_0054',
+        u'amHostHttpAddress': u'localhost:8042',
+        u'startedTime': 1356961057225,
+        u'queue': u'default',
+        u'state': u'RUNNING',
+        u'elapsedTime': 12894,
+        u'finalStatus': u'UNDEFINED',
+        u'diagnostics': u'',
+        u'progress': 100.0,
+        u'trackingUI': u'History',
+        u'id': u'application_1356251510842_0054',
+        u'user': u'test',
         # For when the job is KILLED
-        'startTime': 1356961057226, 'finishTime': 1356961057226,
-        'applicationType': 'MAPREDUCE'
+        u'startTime': 1356961057226,
+        u'finishTime': 1356961057226,
+        u'applicationType': 'MAPREDUCE'
     },
     'application_1356251510842_0009': {
-        u'finishedTime': 1356467118570, u'name': u'oozie:action:T=map-reduce:W=MapReduce-copy2:A=Sleep:ID=0000002-121223003201296-oozie-oozi-W',
-        u'amContainerLogs': u'http://runreal:8042/node/containerlogs/container_1356251510842_0009_01_000001/romain', u'clusterId': 1356251510842,
-        u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0009/jobhistory/job/job_1356251510842_0009', u'amHostHttpAddress': u'runreal:8042',
-        u'startedTime': 1356467081121, u'queue': u'default', u'state': u'FINISHED', u'elapsedTime': 37449, u'finalStatus': u'SUCCEEDED', u'diagnostics': u'',
-        u'progress': 100.0, u'trackingUI': u'History', u'id': u'application_1356251510842_0009', u'user': u'test',
-        'applicationType': 'MAPREDUCE'
-    }
+        u'finishedTime': 1356467118570,
+        u'name': u'oozie:action:T=map-reduce:W=MapReduce-copy2:A=Sleep:ID=0000002-121223003201296-oozie-oozi-W',
+        u'amContainerLogs': u'http://localhost:8042/node/containerlogs/container_1356251510842_0009_01_000001/romain',
+        u'clusterId': 1356251510842,
+        u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0009/jobhistory/job/job_1356251510842_0009',
+        u'amHostHttpAddress': u'localhost:8042',
+        u'startedTime': 1356467081121,
+        u'queue': u'default',
+        u'state': u'FINISHED',
+        u'elapsedTime': 37449,
+        u'finalStatus': u'SUCCEEDED',
+        u'diagnostics': u'',
+        u'progress': 100.0,
+        u'trackingUI': u'History',
+        u'id': u'application_1356251510842_0009',
+        u'user': u'test',
+        u'applicationType': 'MAPREDUCE'
+    },
+    'application_1428442704693_0006': {
+        u'allocatedMB': 4096,
+        u'allocatedVCores': 3,
+        u'amContainerLogs': u'http://localhost:8042/node/containerlogs/container_1428442704693_0006_01_000001/erickt',
+        u'amHostHttpAddress': u'localhost:8042',
+        u'applicationTags': u'',
+        u'applicationType': u'SPARK',
+        u'clusterId': 1428442704693,
+        u'diagnostics': u'',
+        u'elapsedTime': 529040,
+        u'finalStatus': u'UNDEFINED',
+        u'finishedTime': 0,
+        u'id': u'application_1428442704693_0006',
+        u'memorySeconds': 2138468,
+        u'name': u'Spark shell',
+        u'numAMContainerPreempted': 0,
+        u'numNonAMContainerPreempted': 0,
+        u'preemptedResourceMB': 0,
+        u'preemptedResourceVCores': 0,
+        u'progress': 10.0,
+        u'queue': u'root.erickt',
+        u'runningContainers': 3,
+        u'startedTime': 1428443335161,
+        u'state': u'RUNNING',
+        u'trackingUI': u'ApplicationMaster',
+        u'trackingUrl': u'http://localhost:8088/proxy/application_1428442704693_0006/',
+        u'user': u'test',
+        u'vcoreSeconds': 1567,
+    },
+    'application_1428442704693_0007': {
+        u'allocatedMB': -1,
+        u'allocatedVCores': -1,
+        u'applicationTags': u'',
+        u'applicationType': u'YARN',
+        u'clusterId': 1428442704693,
+        u'diagnostics': u'',
+        u'elapsedTime': 4056,
+        u'finalStatus': u'SUCCEEDED',
+        u'finishedTime': 1428454945371,
+        u'id': u'application_1428442704693_0007',
+        u'memorySeconds': 2290,
+        u'name': u'UnmanagedAM',
+        u'numAMContainerPreempted': 0,
+        u'numNonAMContainerPreempted': 0,
+        u'preemptedResourceMB': 0,
+        u'preemptedResourceVCores': 0,
+        u'progress': 100.0,
+        u'queue': u'root.erickt',
+        u'runningContainers': -1,
+        u'startedTime': 1428454941315,
+        u'state': u'FINISHED',
+        u'trackingUI': u'History',
+        u'trackingUrl': u'http://N/A',
+        u'user': u'test',
+        u'vcoreSeconds': 1,
+    },
   }
 
   def __init__(self, oozie_url=None): pass
@@ -498,6 +585,10 @@ class MockResourceManagerApi:
          MockResourceManagerApi.APPS['application_1356251510842_0054'],
          # FINISHED
          MockResourceManagerApi.APPS['application_1356251510842_0009'],
+         # SPARK
+         MockResourceManagerApi.APPS['application_1428442704693_0006'],
+         # YARN
+         MockResourceManagerApi.APPS['application_1428442704693_0007'],
         ]
       }
     }
@@ -745,4 +836,4 @@ def test_make_log_links():
   assert_equal(
       """ pig-job_201307091553_0028.log""",
       LinkJobLogs._make_links(' pig-job_201307091553_0028.log')
-  )
+  )

+ 6 - 2
apps/jobbrowser/src/jobbrowser/yarn_models.py

@@ -77,11 +77,15 @@ class Application(object):
     if not hasattr(self, 'acls'):
       setattr(self, 'acls', {})
 
+    # YARN returns a N/A url if it's not set.
+    if not hasattr(self, 'trackingUrl') or self.trackingUrl == 'http://N/A':
+      self.trackingUrl = None
+
   def kill(self):
     return self.api.kill(self.id)
 
   def filter_tasks(self, *args, **kwargs):
-    pass
+    return []
 
 
 class SparkJob(Application):
@@ -153,7 +157,7 @@ class Job(object):
     setattr(self, 'desiredMaps', None)
     setattr(self, 'finishedReduces', self.reducesCompleted)
     setattr(self, 'desiredReduces', None)
-    setattr(self, 'applicationType', None)
+    setattr(self, 'applicationType', 'MR2')
 
   def kill(self):
     return self.api.kill(self.id)