Browse Source

[jb] Provide links to Hive jobs in Job Browser logs

krish 10 years ago
parent
commit
f8070202a6

+ 36 - 0
apps/jobbrowser/src/jobbrowser/models.py

@@ -23,8 +23,11 @@ import urllib2
 
 from urlparse import urlparse, urlunparse
 
+from django.core.urlresolvers import reverse
 from desktop.lib.view_util import format_duration_in_millis
 from desktop.lib import i18n
+from django.utils.html import escape
+from filebrowser.views import location_to_url
 from hadoop import job_tracker
 from hadoop import confparse
 from hadoop.api.jobtracker.ttypes import JobNotFoundException
@@ -542,6 +545,39 @@ class Cluster(object):
     self.httpPort = self.status.httpPort
 
 
+class LinkJobLogs(object):
+
+  @classmethod
+  def _make_hdfs_links(cls, log):
+    escaped_logs = escape(log)
+    return re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', LinkJobLogs._replace_hdfs_link, escaped_logs)
+
+  @classmethod
+  def _make_mr_links(cls, log):
+    escaped_logs = escape(log)
+    return re.sub('(job_[0-9_]+(/|\.)?)', LinkJobLogs._replace_mr_link, escaped_logs)
+
+  @classmethod
+  def _make_links(cls, log):
+    escaped_logs = escape(log)
+    hdfs_links = re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', LinkJobLogs._replace_hdfs_link, escaped_logs)
+    return re.sub('(job_[0-9_]+(/|\.)?)', LinkJobLogs._replace_mr_link, hdfs_links)
+
+  @classmethod
+  def _replace_hdfs_link(self, match):
+    try:
+      return '<a href="%s" target="_blank">%s</a>' % (location_to_url(match.group(0), strict=False), match.group(0))
+    except:
+      return match.group(0)
+
+  @classmethod
+  def _replace_mr_link(self, match):
+    try:
+      return '<a href="%s" target="_blank">%s</a>' % (reverse('jobbrowser.views.single_job', kwargs={'job': match.group(0)}), match.group(0))
+    except:
+      return match.group(0)
+
+
 def get_jobconf(jt, jobid):
   """
   Returns a dict representation of the jobconf for the job corresponding

+ 1 - 1
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/utils.js

@@ -33,7 +33,7 @@ function initLogsElement(element) {
 function appendAndScroll(element, logs) {
   var newLines = logs.split("\n").slice(element.text().split("\n").length);
   if (newLines.length > 0) {
-    element.text(element.text() + newLines.join("\n") + "\n");
+    element.html(element.text() + newLines.join("\n") + "\n");
   }
   if (element.data("logsAtEnd")) {
     element.scrollTop(element[0].scrollHeight - element.height());

+ 4 - 4
apps/jobbrowser/src/jobbrowser/templates/attempt_logs.mako

@@ -87,28 +87,28 @@ ${ comps.menubar() }
                       % if not log_diagnostic:
                         <pre>-- empty --</pre>
                       % else:
-                        <pre>${format_log(log_diagnostic)}</pre>
+                        <pre></pre>
                       % endif
                   </div>
                   <div class="tab-pane ${ first_log_tab == 1 and 'active' or '' }" id="logsStdOut">
                       % if not log_stdout:
                         <pre>-- empty --</pre>
                       % else:
-                        <pre>${format_log(log_stdout)}</pre>
+                        <pre></pre>
                       % endif
                   </div>
                   <div class="tab-pane ${ first_log_tab == 2 and 'active' or '' }" id="logsStdErr">
                       % if not log_stderr:
                         <pre>-- empty --</pre>
                       % else:
-                        <pre>${format_log(log_stderr)}</pre>
+                        <pre></pre>
                       % endif
                   </div>
                   <div class="tab-pane ${ first_log_tab == 3 and 'active' or '' }" id="logsSysLog">
                       % if not log_syslog:
                         <pre>-- empty --</pre>
                       % else:
-                        <pre>${format_log(log_syslog)}</pre>
+                        <pre></pre>
                       % endif
                   </div>
                 </div>

+ 67 - 1
apps/jobbrowser/src/jobbrowser/tests.py

@@ -37,7 +37,7 @@ from oozie.models import Workflow
 
 from jobbrowser import models, views
 from jobbrowser.conf import SHARE_JOBS
-from jobbrowser.models import can_view_job, can_modify_job, Job
+from jobbrowser.models import can_view_job, can_modify_job, Job, LinkJobLogs
 
 
 LOG = logging.getLogger(__name__)
@@ -680,3 +680,69 @@ class HistoryServerApi(MockMapreduce2Api):
               u'avgShuffleTime': 1421, u'queue': u'default', u'killedReduceAttempts': 0, u'failedMapAttempts': 0
           }
       }
+
+
+def test_make_log_links():
+  """
+   Unit test for models.LinkJobLogs._make_links
+  """
+
+  # FileBrowser
+  assert_equal(
+      """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a>  &lt;dir&gt;""",
+      LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/tmp  <dir>')
+  )
+  assert_equal(
+      """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a>&lt;dir&gt;""",
+      LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/tmp<dir>')
+  )
+  assert_equal(
+      """output: <a href="/filebrowser/view/user/romain/tmp" target="_blank">/user/romain/tmp</a>  &lt;dir&gt;""",
+      LinkJobLogs._make_links('output: /user/romain/tmp  <dir>')
+  )
+  assert_equal(
+      'Successfully read 3760 records (112648 bytes) from: &quot;<a href="/filebrowser/view/user/hue/pig/examples/data/midsummer.txt" target="_blank">/user/hue/pig/examples/data/midsummer.txt</a>&quot;',
+      LinkJobLogs._make_links('Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"')
+  )
+  assert_equal(
+      'data,upper_case  MAP_ONLY  <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>,',
+      LinkJobLogs._make_links('data,upper_case  MAP_ONLY  hdfs://localhost:8020/user/romain/out/fffff,')
+  )
+  assert_equal(
+      'MAP_ONLY  <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013',
+      LinkJobLogs._make_links('MAP_ONLY  hdfs://localhost:8020/user/romain/out/fffff\n2013')
+  )
+  assert_equal(
+      ' <a href="/filebrowser/view/jobs.tsv" target="_blank">/jobs.tsv</a> ',
+      LinkJobLogs._make_links(' /jobs.tsv ')
+  )
+  assert_equal(
+      '<a href="/filebrowser/view/user/romain/job_pos_2012.tsv" target="_blank">hdfs://localhost:8020/user/romain/job_pos_2012.tsv</a>',
+      LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/job_pos_2012.tsv')
+  )
+
+  # JobBrowser
+  assert_equal(
+      """<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+      LinkJobLogs._make_links('job_201306261521_0058')
+  )
+  assert_equal(
+      """Hadoop Job IDs executed by Pig: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+      LinkJobLogs._make_links('Hadoop Job IDs executed by Pig: job_201306261521_0058')
+  )
+  assert_equal(
+      """MapReduceLauncher  - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+      LinkJobLogs._make_links('MapReduceLauncher  - HadoopJobId: job_201306261521_0058')
+  )
+  assert_equal(
+      """- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+      LinkJobLogs._make_links('- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058')
+  )
+  assert_equal(
+      """ Logging error messages to: job_201307091553_0028/attempt_201307091553_002""",
+      LinkJobLogs._make_links(' Logging error messages to: job_201307091553_0028/attempt_201307091553_002')
+  )
+  assert_equal(
+      """ pig-job_201307091553_0028.log""",
+      LinkJobLogs._make_links(' pig-job_201307091553_0028.log')
+  )

+ 7 - 4
apps/jobbrowser/src/jobbrowser/views.py

@@ -43,7 +43,7 @@ from hadoop.yarn.clients import get_log_client
 
 from jobbrowser import conf
 from jobbrowser.api import get_api, ApplicationNotRunning, JobExpired
-from jobbrowser.models import Job, JobLinkage, Tracker, Cluster, can_view_job, can_modify_job
+from jobbrowser.models import Job, JobLinkage, Tracker, Cluster, can_view_job, can_modify_job, LinkJobLogs
 
 import urllib2
 
@@ -290,7 +290,7 @@ def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=0
     except:
       pass
 
-  response = {'log': log, 'debug': debug_info}
+  response = {'log': LinkJobLogs._make_hdfs_links(log), 'debug': debug_info}
 
   return JsonResponse(response)
 
@@ -441,9 +441,12 @@ def single_task_attempt_logs(request, job, taskid, attemptid):
 
   if request.GET.get('format') == 'python':
     return context
-  elif request.GET.get('format') == 'json':
+  else:
+    context['logs'] = [LinkJobLogs._make_links(log) for i, log in enumerate(logs)]
+
+  if request.GET.get('format') == 'json':
     response = {
-      "logs": logs,
+      "logs": context['logs'],
       "isRunning": job.status.lower() in ('running', 'pending', 'prep')
     }
     return JsonResponse(response)

+ 2 - 22
apps/pig/src/pig/api.py

@@ -21,13 +21,12 @@ import re
 import time
 
 from django.core.urlresolvers import reverse
-from django.utils.html import escape
 from django.utils.translation import ugettext as _
 
 from desktop.lib.i18n import smart_str
 from desktop.lib.view_util import format_duration_in_millis
-from filebrowser.views import location_to_url
 from jobbrowser.views import job_single_logs
+from jobbrowser.models import LinkJobLogs
 from liboozie.oozie_api import get_oozie
 from oozie.models import Workflow, Pig
 from oozie.views.editor import _submit_workflow
@@ -159,7 +158,7 @@ class OozieApi(object):
           data = job_single_logs(request, **{'job': action.externalId})
           if data:
             matched_logs = self._match_logs(data)
-            logs[action.name] = self._make_links(matched_logs)
+            logs[action.name] = LinkJobLogs._make_links(matched_logs)
             is_really_done = OozieApi.RE_LOG_END.search(data['logs'][1]) is not None
 
       except Exception, e:
@@ -195,25 +194,6 @@ class OozieApi(object):
       i = logs.index(group.group(1)) + len(group.group(1))
       return logs[i:].strip()
 
-  @classmethod
-  def _make_links(cls, log):
-    escaped_logs = escape(log)
-    hdfs_links = re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', OozieApi._make_hdfs_link, escaped_logs)
-    return re.sub('(job_[0-9_]+(/|\.)?)', OozieApi._make_mr_link, hdfs_links)
-
-  @classmethod
-  def _make_hdfs_link(self, match):
-    try:
-      return '<a href="%s" target="_blank">%s</a>' % (location_to_url(match.group(0), strict=False), match.group(0))
-    except:
-      return match.group(0)
-
-  @classmethod
-  def _make_mr_link(self, match):
-    try:
-      return '<a href="%s" target="_blank">%s</a>' % (reverse('jobbrowser.views.single_job', kwargs={'job': match.group(0)}), match.group(0))
-    except:
-      return match.group(0)
 
   def massaged_jobs_for_json(self, request, oozie_jobs, hue_jobs):
     jobs = []

+ 0 - 61
apps/pig/src/pig/tests.py

@@ -60,67 +60,6 @@ def create_script(user, xattrs=None):
     attrs.update(xattrs)
   return create_or_update_script(**attrs)
 
-def test_make_log_links():
-  # FileBrowser
-  assert_equal(
-      """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a>  &lt;dir&gt;""",
-      OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp  <dir>')
-  )
-  assert_equal(
-      """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a>&lt;dir&gt;""",
-      OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp<dir>')
-  )
-  assert_equal(
-      """output: <a href="/filebrowser/view/user/romain/tmp" target="_blank">/user/romain/tmp</a>  &lt;dir&gt;""",
-      OozieApi._make_links('output: /user/romain/tmp  <dir>')
-  )
-  assert_equal(
-      'Successfully read 3760 records (112648 bytes) from: &quot;<a href="/filebrowser/view/user/hue/pig/examples/data/midsummer.txt" target="_blank">/user/hue/pig/examples/data/midsummer.txt</a>&quot;',
-      OozieApi._make_links('Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"')
-  )
-  assert_equal(
-      'data,upper_case  MAP_ONLY  <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>,',
-      OozieApi._make_links('data,upper_case  MAP_ONLY  hdfs://localhost:8020/user/romain/out/fffff,')
-  )
-  assert_equal(
-      'MAP_ONLY  <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013',
-      OozieApi._make_links('MAP_ONLY  hdfs://localhost:8020/user/romain/out/fffff\n2013')
-  )
-  assert_equal(
-      ' <a href="/filebrowser/view/jobs.tsv" target="_blank">/jobs.tsv</a> ',
-      OozieApi._make_links(' /jobs.tsv ')
-  )
-  assert_equal(
-      '<a href="/filebrowser/view/user/romain/job_pos_2012.tsv" target="_blank">hdfs://localhost:8020/user/romain/job_pos_2012.tsv</a>',
-      OozieApi._make_links('hdfs://localhost:8020/user/romain/job_pos_2012.tsv')
-  )
-
-  # JobBrowser
-  assert_equal(
-      """<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
-      OozieApi._make_links('job_201306261521_0058')
-  )
-  assert_equal(
-      """Hadoop Job IDs executed by Pig: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
-      OozieApi._make_links('Hadoop Job IDs executed by Pig: job_201306261521_0058')
-  )
-  assert_equal(
-      """MapReduceLauncher  - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
-      OozieApi._make_links('MapReduceLauncher  - HadoopJobId: job_201306261521_0058')
-  )
-  assert_equal(
-      """- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
-      OozieApi._make_links('- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058')
-  )
-  assert_equal(
-      """ Logging error messages to: job_201307091553_0028/attempt_201307091553_002""",
-      OozieApi._make_links(' Logging error messages to: job_201307091553_0028/attempt_201307091553_002')
-  )
-  assert_equal(
-      """ pig-job_201307091553_0028.log""",
-      OozieApi._make_links(' pig-job_201307091553_0028.log')
-  )
-
 
 class TestMock(TestPigBase):