Bläddra i källkod

HUE-1158 [jb] MR2 Incorrect logs shown

Romain Rigaux 12 år sedan
förälder
incheckning
2c2648d711

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/job.mako

@@ -108,7 +108,7 @@ ${ commonheader(_('Job: %(jobId)s - Job Browser') % dict(jobId=job.jobId_short),
                     </li>
                     <li class="nav-header">${_('Logs')}</li>
                     <li><a href="${ url('jobbrowser.views.job_single_logs', job=job.jobId) }"><i class="icon-tasks"></i> ${_('Logs')}</a></li>
-                    % if job.status.lower() == 'running' or job.status.lower() == 'pending':
+                    % if not job.is_mr2 and (job.status.lower() in ('running', 'pending')):
                         <li class="nav-header">${_('Kill Job')}</li>
                         <li>
                           <a href="#" title="${_('Kill this job')}" onclick="$('#kill-job').submit()">${_('Kill this job')}</a>

+ 6 - 3
apps/jobbrowser/src/jobbrowser/urls.py

@@ -24,10 +24,7 @@ urlpatterns = patterns('jobbrowser.views',
   url(r'^jobs/(?P<job>\w+)$','single_job', name='single_job'),
   url(r'^jobs/(?P<job>\w+)/counters$', 'job_counters', name='job_counters'),
   url(r'^jobs/(?P<job>\w+)/kill$', 'kill_job', name='kill_job'),
-  url(r'^jobs/(?P<job>\w+)/setpriority$', 'set_job_priority', name='set_job_priority'), #? used
   url(r'^jobs/(?P<job>\w+)/single_logs$', 'job_single_logs', name='job_single_logs'),
-  url(r'^jobs/(?P<job>\w+)/job_attempt_logs/(?P<attempt_index>\d+)$', 'job_attempt_logs', name='job_attempt_logs'),
-  url(r'^jobs/(?P<job>\w+)/job_attempt_logs_json/(?P<attempt_index>\d+)/(?P<name>\w+)?/(?P<offset>\d+)?$', 'job_attempt_logs_json', name='job_attempt_logs_json'), # MR2
   url(r'^jobs/(?P<job>\w+)/tasks$','tasks', name='tasks'),
   url(r'^jobs/(?P<job>\w+)/tasks/(?P<taskid>\w+)$', 'single_task', name='single_task'), # TODO s/single// ?
   url(r'^jobs/(?P<job>\w+)/tasks/(?P<taskid>\w+)/attempts/(?P<attemptid>\w+)$', 'single_task_attempt', name='single_task_attempt'),
@@ -35,7 +32,13 @@ urlpatterns = patterns('jobbrowser.views',
   url(r'^jobs/(?P<job>\w+)/tasks/(?P<taskid>\w+)/attempts/(?P<attemptid>\w+)/logs$', 'single_task_attempt_logs', name='single_task_attempt_logs'),
   url(r'^jobs/(\w+)/tasks/(\w+)/attempts/(?P<attemptid>\w+)/kill$', 'kill_task_attempt', name='kill_task_attempt'),
   url(r'^trackers/(?P<trackerid>.+)$', 'single_tracker', name='single_tracker'),
+
+  # MR2 specific
+  url(r'^jobs/(?P<job>\w+)/job_attempt_logs/(?P<attempt_index>\d+)$', 'job_attempt_logs', name='job_attempt_logs'),
+  url(r'^jobs/(?P<job>\w+)/job_attempt_logs_json/(?P<attempt_index>\d+)/(?P<name>\w+)?/(?P<offset>\d+)?$', 'job_attempt_logs_json', name='job_attempt_logs_json'),
+
   # Unused
+  url(r'^jobs/(?P<job>\w+)/setpriority$', 'set_job_priority', name='set_job_priority'),
   url(r'^trackers$', 'trackers', name='trackers'),
   url(r'^clusterstatus$', 'clusterstatus', name='clusterstatus'),
   url(r'^queues$', 'queues', name='queues'),

+ 7 - 8
apps/jobbrowser/src/jobbrowser/views.py

@@ -270,10 +270,6 @@ def single_task_attempt_logs(request, job, taskid, attemptid):
   jt = get_api(request.user, request.jt)
 
   job_link = jt.get_job_link(job.jobId)
-
-  if job_link.is_mr2:
-    return job_attempt_logs(request, job=job.jobId)
-
   task = job_link.get_task(taskid)
 
   try:
@@ -285,17 +281,20 @@ def single_task_attempt_logs(request, job, taskid, attemptid):
 
   try:
     # Add a diagnostic log
-    diagnostic_log = ", ".join(task.diagnosticMap[attempt.attemptId])
-    logs = [ diagnostic_log ]
+    if job_link.is_mr2:
+      diagnostic_log = attempt.diagnostics
+    else:
+      diagnostic_log =  ", ".join(task.diagnosticMap[attempt.attemptId])
+    logs = [diagnostic_log]
     # Add remaining logs
-    logs += [ section.strip() for section in attempt.get_task_log() ]
+    logs += [section.strip() for section in attempt.get_task_log()]
     log_tab = [i for i, log in enumerate(logs) if log]
     if log_tab:
       first_log_tab = log_tab[0]
   except TaskTrackerNotFoundException:
     # Four entries,
     # for diagnostic, stdout, stderr and syslog
-    logs = [ _("Failed to retrieve log. TaskTracker not found.") ] * 4
+    logs = [_("Failed to retrieve log. TaskTracker not found.")] * 4
 
   context = {
       "attempt": attempt,

+ 25 - 0
apps/jobbrowser/src/jobbrowser/yarn_models.py

@@ -19,6 +19,8 @@ import logging
 import re
 import time
 
+from lxml import html
+
 from desktop.lib.view_util import format_duration_in_millis
 
 from jobbrowser.models import format_unixtime_ms
@@ -198,6 +200,8 @@ class Attempt:
     setattr(self, 'shuffleFinishTimeFormatted', None)
     setattr(self, 'sortFinishTimeFormatted', None)
     setattr(self, 'mapFinishTimeFormatted', None)
+    if not hasattr(self, 'diagnostics'):
+      self.diagnostics = ''
 
   @property
   def counters(self):
@@ -205,6 +209,27 @@ class Attempt:
       self._counters = self.task.job.api.task_attempt_counters(self.task.jobId, self.task.id, self.id)['jobCounters']
     return self._counters
 
+  def get_task_log(self, offset=0):
+    logs = []
+
+    attempt = self.task.job.job_attempts['jobAttempt'][0]
+    log_link = attempt['logsLink']
+
+    for name in ('stdout', 'stderr', 'syslog'):
+      link = '/%s/' % name
+      if int(offset) >= 0:
+        link += '?start=%s' % offset
+
+      try:
+        log_link = re.sub('job_[^/]+', self.id, log_link)
+        log = html.parse(log_link + link).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
+      except Exception, e:
+        log = _('Failed to retrieve log: %s') % e
+
+      logs.append(log)
+
+    return logs + [''] * (3 - len(logs))
+
 
 class Container: