Эх сурвалжийг харах

HUE-2094 [jb] Add more metadata when viewing Spark application details

Scrapes metadata from oozie history browser.
Karissa McKelvey 11 жил өмнө
parent
commit
11c7ef6b28

+ 2 - 1
apps/jobbrowser/src/jobbrowser/api.py

@@ -30,6 +30,7 @@ import hadoop.yarn.node_manager_api as node_manager_api
 from jobbrowser.conf import SHARE_JOBS
 from jobbrowser.models import Job, JobLinkage, TaskList, Tracker
 from jobbrowser.yarn_models import Application, Job as YarnJob, KilledJob as KilledYarnJob, Container
+from jobbrowser.yarn_models import SparkJob
 from hadoop.cluster import get_next_ha_mrcluster, get_next_ha_yarncluster
 from desktop.lib.exceptions_renderable import PopupException
 
@@ -264,7 +265,7 @@ class YarnApi(JobBrowserApi):
         return KilledYarnJob(self.resource_manager_api, job)
 
       if job['applicationType'] == 'SPARK':
-        job = Application(job)
+        job = SparkJob(job)
       else:
         # MR id, assume 'applicationType': 'MAPREDUCE'
         jobid = jobid.replace('application', 'job')

+ 73 - 11
apps/jobbrowser/src/jobbrowser/templates/job.mako

@@ -22,7 +22,6 @@
   from jobbrowser.views import format_counter_name
   from filebrowser.views import location_to_url
   from desktop.views import commonheader, commonfooter
-
   from django.template.defaultfilters import urlencode
   from django.utils.translation import ugettext as _
 %>
@@ -107,7 +106,7 @@ ${ comps.menubar() }
     <div class="span2">
       <div class="sidebar-nav" style="padding-top: 0">
         <ul class="nav nav-list">
-          <li class="nav-header">${_('Job ID')}</li>
+          <li class="nav-header">${_('App ID')}</li>
           <li class="white hellipsify">${job.jobId_short}</li>
           <li class="nav-header">${_('User')}</li>
           <li class="white">${job.user}</li>
@@ -115,12 +114,8 @@ ${ comps.menubar() }
           <li class="white" id="jobStatus">&nbsp;</li>
           <li class="nav-header">${_('Logs')}</li>
           <li><a href="${job.trackingUrl }" target="_blank"><i class="fa fa-tasks"></i> ${_('Logs')}</a></li>
-          % if not job.is_retired:
-          <li class="nav-header">${_('Maps')}</li>
-          <li class="white" id="jobMaps">&nbsp;</li>
-          <li class="nav-header">${_('Reduces')}</li>
-          <li class="white" id="jobReduces">&nbsp;</li>
-          % endif
+          <li class="nav-header">${_('Progress')}</li>
+          <li class="white">${job.progress}%</li>
           <li class="nav-header">${_('Duration')}</li>
           <li class="white">${job.durationFormatted}</li>
           <li class="nav-header killJob">${_('Actions')}</li>
@@ -132,9 +127,75 @@ ${ comps.menubar() }
       <div class="card card-small">
         <h1 class="card-heading simple">${_(job.name)}</h1>
         <div class="card-body">
-          <p>
-            <a href="${job.trackingUrl}" id="tracking-link" target="_blank">${_('Go to spark job list at %s' % job.trackingUrl)}</a>
-          </p>
+          <ul class="nav nav-tabs">
+            <li  class="active"><a href="#metadata" data-toggle="tab">${_('Metadata')}</a></li>
+            % if job.scrapedData.get('metrics'):
+              <li><a href="#metrics" data-toggle="tab">${_('Metrics')}</a></li>
+            % endif
+          </ul>
+          <div class="tab-content">
+            <div class="tab-pane active" id="metadata">
+              <table class="table table-condensed">
+                <thead>
+                  <th>${_('Name')}</th>
+                  <th>${_('Value')}</th>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td>${_('Jobs')}</td>
+                    <td><a href="${job.trackingUrl}">${job.trackingUrl}</a></td>
+                  </tr>
+                  <tr>
+                    <td>${_('Host')}</td>
+                    <td><a href="http://${job.amHostHttpAddress}">http://${job.amHostHttpAddress}</a></td>
+                  </tr>
+                  <tr>
+                    <td>${_('Queue Name')}</td>
+                    <td>${job.queueName}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Started')}</td>
+                    <td>${job.startTimeFormatted}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Finished')}</td>
+                    <td>${job.finishTimeFormatted}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Pre-empted Resource VCores')}</td>
+                    <td>${job.preemptedResourceVCores}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('VCore seconds')}</td>
+                    <td>${job.vcoreSeconds}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Memory seconds')}</td>
+                    <td>${job.memorySeconds}</td>
+                  </tr>
+                  <tr>
+                    <td>${_('Diagnostics')}</td>
+                    <td>${job.diagnostics}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="tab-pane" id="metrics">
+              <table class="table table-condensed">
+                <thead>
+                  <th>${_('Metric')}</th>
+                  <th>${_('Value')}</th>
+                </thead>
+                <tbody>
+                % for metric in job.scrapedData.get('metrics', []):
+                  <tr>
+                    <td>${_(metric['header'])}</td>
+                    <td>${metric['value']}</td>
+                  </tr>
+                % endfor
+                </tbody>
+              </table>
+            </div>
         </div>
       </div>
     </div>
@@ -191,6 +252,7 @@ ${ comps.menubar() }
         <h1 class="card-heading simple">${_(job.name)}</h1>
           <div class="card-body">
             <p>
+
               <ul class="nav nav-tabs">
                 % if job.is_mr2:
                 <li class="active"><a href="#attempts" data-toggle="tab">${_('Attempts')}</a></li>

+ 1 - 1
apps/jobbrowser/src/jobbrowser/views.py

@@ -60,7 +60,7 @@ def check_job_permission(view_func):
       # reverse() seems broken, using request.path but beware, it discards GET and POST info
       return job_not_assigned(request, jobid, request.path)
     except Exception, e:
-      raise PopupException(_('Could not find job %s.') % jobid, detail=e)
+       raise PopupException(_('Could not find job %s.') % jobid, detail=e)
 
     if not conf.SHARE_JOBS.get() and not request.user.is_superuser \
         and job.user != request.user.username and not can_view_job(request.user.username, job):

+ 42 - 1
apps/jobbrowser/src/jobbrowser/yarn_models.py

@@ -19,6 +19,7 @@ import logging
 import re
 import time
 import urlparse
+import urllib2
 
 from lxml import html
 
@@ -35,7 +36,7 @@ from jobbrowser.models import format_unixtime_ms
 LOGGER = logging.getLogger(__name__)
 
 
-class Application:
+class Application(object):
 
   def __init__(self, attrs):
     for attr in attrs.keys():
@@ -65,6 +66,7 @@ class Application:
     setattr(self, 'durationInMillis', finishTime - self.startedTime)
     setattr(self, 'startTimeMs', self.startedTime)
     setattr(self, 'startTimeFormatted',  format_unixtime_ms(self.startedTime))
+    setattr(self, 'finishTimeFormatted',  format_unixtime_ms(finishTime))
     setattr(self, 'finishedMaps', None)
     setattr(self, 'desiredMaps', None)
     setattr(self, 'finishedReduces', None)
@@ -74,6 +76,44 @@ class Application:
     if not hasattr(self, 'acls'):
       setattr(self, 'acls', {})
 
+class SparkJob(Application):
+
+  def __init__(self, job):
+    super(SparkJob, self).__init__(job)
+    self._scrape()
+
+  def _history_application_metrics(self, html_doc):
+    metrics = []
+    root = html.fromstring(html_doc)
+    tables = root.findall('.//table')
+    metrics_table = tables[2].findall('.//tr')
+    for tr in metrics_table:
+        header = tr.find('.//th')
+        value = tr.findall('.//td')
+        if value:
+          header = header.text.strip().replace(':', '')
+          value = value[0].text.strip()
+          metrics.append({
+            'header': header,
+            'value': value
+          })
+    return metrics
+
+  def _scrape(self):
+    # XXX: we have to scrape the tracking URL directly because
+    # spark jobs don't have a JSON api via YARN or app server
+    # see YARN-1530, SPARK-1537 for progress on these apis
+    self.scrapedData = {}
+    try:
+      res = urllib2.urlopen(self.trackingUrl)
+      html_doc = res.read()
+      if self.trackingUI == 'History':
+        self.scrapedData['metrics'] = self._history_application_metrics(html_doc)
+    except Exception, e:
+      # Prevent a nosedive. Don't create metrics if api changes or url is unreachable.
+      self.scrapedData['metrics'] = []
+
+
 class Job(object):
 
   def __init__(self, api, attrs):
@@ -326,3 +366,4 @@ class Container:
     setattr(self, 'maxMapTasks', None)
     setattr(self, 'maxReduceTasks', None)
     setattr(self, 'taskReports', None)
+