瀏覽代碼

HUE-1165 [pig] Logs with MR2

Romain Rigaux 12 年之前
父節點
當前提交
72a17ef

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/tasktracker.mako

@@ -82,4 +82,4 @@ ${ commonheader(_('Tracker: %(trackerId)s') % dict(trackerId=tracker.trackerId),
     % endif
 </div>
 
-${ commonfooter(messages) | n,unicode }
+${ commonfooter(messages) | n,unicode }

+ 25 - 15
apps/jobbrowser/src/jobbrowser/tests.py

@@ -478,7 +478,7 @@ class TestMapReduce2:
 
   def test_running_job(self):
     response = self.c.get('/jobbrowser/jobs/application_1356251510842_0054')
-    assert_equal(response.context['job'].jobId, 'application_1356251510842_0054')
+    assert_equal(response.context['job'].jobId, 'job_1356251510842_0054')
 
     response = self.c.get('/jobbrowser/jobs/job_1356251510842_0054')
     assert_false('job' in response.context)
@@ -492,28 +492,38 @@ class TestMapReduce2:
 
 
 class MockResourceManagerApi:
+  APPS = {
+    'application_1356251510842_0054': {u'finishedTime': 1356961070119, u'name': u'oozie:launcher:T=map-reduce:W=MapReduce-copy:A=Sleep:ID=0000004-121223003201296-oozie-oozi-W',
+    u'amContainerLogs': u'http://runreal:8042/node/containerlogs/container_1356251510842_0054_01_000001/romain', u'clusterId': 1356251510842,
+    u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0054/jobhistory/job/job_1356251510842_0054', u'amHostHttpAddress': u'runreal:8042',
+    u'startedTime': 1356961057225, u'queue': u'default', u'state': u'RUNNING', u'elapsedTime': 12894, u'finalStatus': u'UNDEFINED', u'diagnostics': u'',
+    u'progress': 100.0, u'trackingUI': u'History', u'id': u'application_1356251510842_0054', u'user': u'romain'},
+  'application_1356251510842_0009': {u'finishedTime': 1356467118570, u'name': u'oozie:action:T=map-reduce:W=MapReduce-copy2:A=Sleep:ID=0000002-121223003201296-oozie-oozi-W',
+    u'amContainerLogs': u'http://runreal:8042/node/containerlogs/container_1356251510842_0009_01_000001/romain', u'clusterId': 1356251510842,
+    u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0009/jobhistory/job/job_1356251510842_0009', u'amHostHttpAddress': u'runreal:8042',
+    u'startedTime': 1356467081121, u'queue': u'default', u'state': u'FINISHED', u'elapsedTime': 37449, u'finalStatus': u'SUCCEEDED', u'diagnostics': u'',
+    u'progress': 100.0, u'trackingUI': u'History', u'id': u'application_1356251510842_0009', u'user': u'romain'}
+  }
 
   def __init__(self, oozie_url=None): pass
 
   def apps(self, **kwargs):
     return {
-      u'apps':
-        {u'app': [
-           # RUNNING application_1356251510842_0054
-           {u'finishedTime': 1356961070119, u'name': u'oozie:launcher:T=map-reduce:W=MapReduce-copy:A=Sleep:ID=0000004-121223003201296-oozie-oozi-W',
-            u'amContainerLogs': u'http://runreal:8042/node/containerlogs/container_1356251510842_0054_01_000001/romain', u'clusterId': 1356251510842,
-            u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0054/jobhistory/job/job_1356251510842_0054', u'amHostHttpAddress': u'runreal:8042',
-            u'startedTime': 1356961057225, u'queue': u'default', u'state': u'RUNNING', u'elapsedTime': 12894, u'finalStatus': u'UNDEFINED', u'diagnostics': u'',
-            u'progress': 100.0, u'trackingUI': u'History', u'id': u'application_1356251510842_0054', u'user': u'romain'},
-           # FINISHED application_1356251510842_0009
-           {u'finishedTime': 1356467118570, u'name': u'oozie:action:T=map-reduce:W=MapReduce-copy2:A=Sleep:ID=0000002-121223003201296-oozie-oozi-W',
-            u'amContainerLogs': u'http://runreal:8042/node/containerlogs/container_1356251510842_0009_01_000001/romain', u'clusterId': 1356251510842,
-            u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0009/jobhistory/job/job_1356251510842_0009', u'amHostHttpAddress': u'runreal:8042',
-            u'startedTime': 1356467081121, u'queue': u'default', u'state': u'FINISHED', u'elapsedTime': 37449, u'finalStatus': u'SUCCEEDED', u'diagnostics': u'',
-            u'progress': 100.0, u'trackingUI': u'History', u'id': u'application_1356251510842_0009', u'user': u'romain'}]
+     'apps': {
+       'app': [
+         # RUNNING
+         MockResourceManagerApi.APPS['application_1356251510842_0054'],
+         # FINISHED
+         MockResourceManagerApi.APPS['application_1356251510842_0009'],
+        ]
       }
     }
 
+  def app(self, job_id):
+    return {
+      u'app': MockResourceManagerApi.APPS[job_id]
+    }
+
 
 class MockMapreduce2Api(object):
   """

+ 7 - 5
apps/jobbrowser/src/jobbrowser/views.py

@@ -168,10 +168,9 @@ def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=0
 
 @check_job_permission
 def job_single_logs(request, job):
-
-  if job.is_mr2:
-    return job_attempt_logs(request, job=job.jobId)
-
+  """
+  Try to smartly detect the most useful task attempt (e.g. Oozie launcher, failed task) and get its MR logs.
+  """
   def cmp_exec_time(task1, task2):
     return cmp(task1.execStartTimeMs, task2.execStartTimeMs)
 
@@ -350,7 +349,10 @@ def trackers(request):
 def single_tracker(request, trackerid):
   jt = get_api(request.user, request.jt)
 
-  tracker = jt.get_tracker(trackerid)
+  try:
+    tracker = jt.get_tracker(trackerid)
+  except Exception, e:
+    raise PopupException(_('The container disappears as soon as the job finishes.'), detail=e)
   return render("tasktracker.mako", request, {'tracker':tracker})
 
 def clusterstatus(request):

+ 2 - 1
apps/jobbrowser/src/jobbrowser/yarn_models.py

@@ -202,9 +202,10 @@ class Attempt:
 
   def get_task_log(self, offset=0):
     logs = []
-
     attempt = self.task.job.job_attempts['jobAttempt'][0]
     log_link = attempt['logsLink']
+    # Get MR task logs
+    log_link = log_link.replace(attempt['containerId'], self.assignedContainerId)
 
     for name in ('stdout', 'stderr', 'syslog'):
       link = '/%s/' % name

+ 10 - 8
apps/pig/src/pig/api.py

@@ -45,8 +45,8 @@ class OozieApi:
   """
   WORKFLOW_NAME = 'pig-app-hue-script'
   RE_LOG_END = re.compile('(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed <<<)')
-  RE_LOG_START_RUNNING = re.compile('>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?\n(.+?)(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed)', re.M | re.DOTALL)
-  RE_LOG_START_FINISHED = re.compile('(>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?)\n', re.M | re.DOTALL)
+  RE_LOG_START_RUNNING = re.compile('>>> Invoking Pig command line now >>>(.+?)(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed)', re.M | re.DOTALL)
+  RE_LOG_START_FINISHED = re.compile('(>>> Invoking Pig command line now >>>)', re.M | re.DOTALL)
   MAX_DASHBOARD_JOBS = 100
 
   def __init__(self, fs, user):
@@ -121,9 +121,9 @@ class OozieApi:
     for action in oozie_workflow.get_working_actions():
       try:
         if action.externalId:
-          log = job_single_logs(request, **{'job': action.externalId})
-          if log:
-            logs[action.name] = self._match_logs(log['logs'][1])
+          data = job_single_logs(request, **{'job': action.externalId})
+          if data:
+            logs[action.name] = self._match_logs(data)
       except Exception, e:
         LOG.error('An error happen while watching the demo running: %(error)s' % {'error': e})
 
@@ -144,14 +144,16 @@ class OozieApi:
 
     return logs, workflow_actions
 
-  def _match_logs(self, logs):
+  def _match_logs(self, data):
     """Difficult to match multi lines of text"""
+    logs = data['logs'][1]
+
     if OozieApi.RE_LOG_END.search(logs):
-      return re.search(OozieApi.RE_LOG_START_RUNNING, logs).group(1)
+      return re.search(OozieApi.RE_LOG_START_RUNNING, logs).group(1).strip()
     else:
       group = re.search(OozieApi.RE_LOG_START_FINISHED, logs)
       i = logs.index(group.group(1)) + len(group.group(1))
-      return logs[i:]
+      return logs[i:].strip()
 
   def massaged_jobs_for_json(self, request, oozie_jobs, hue_jobs):
     jobs = []

+ 1 - 1
apps/pig/src/pig/conf.py

@@ -43,7 +43,7 @@ def config_validator(user):
 
   if not 'test' in sys.argv: # Avoid tests hanging
     status = get_oozie_status()
-  
+
     if 'NORMAL' not in status:
       res.append((NICE_NAME, _("The app won't work without a running Oozie server")))