Browse Source

HUE-1150 [pig] Links to inputs/outputs/MR jobs

Romain Rigaux 12 years ago
parent
commit
144090d

+ 24 - 1
apps/pig/src/pig/api.py

@@ -14,6 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from filebrowser.views import location_to_url
 
 try:
   import json
@@ -24,6 +25,7 @@ import re
 import time
 
 from django.core.urlresolvers import reverse
+from django.utils.html import escape
 from django.utils.translation import ugettext as _
 
 from jobbrowser.views import job_single_logs
@@ -123,7 +125,8 @@ class OozieApi:
         if action.externalId:
           data = job_single_logs(request, **{'job': action.externalId})
           if data:
-            logs[action.name] = self._match_logs(data)
+            matched_logs = self._match_logs(data)
+            logs[action.name] = self._make_links(matched_logs)
       except Exception, e:
         LOG.error('An error happen while watching the demo running: %(error)s' % {'error': e})
 
@@ -155,6 +158,26 @@ class OozieApi:
       i = logs.index(group.group(1)) + len(group.group(1))
       return logs[i:].strip()
 
+  @classmethod
+  def _make_links(cls, log):
+    escaped_logs = escape(log)
+    hdfs_links = re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', OozieApi._make_hdfs_link, escaped_logs)
+    return re.sub('job_[a-z0-9_]*', OozieApi._make_mr_link, hdfs_links)
+
+  @classmethod
+  def _make_hdfs_link(self, match):
+    try:
+      return '<a href="%s" target="_blank">%s</a>' % (location_to_url(match.group(0), strict=False), match.group(0))
+    except:
+      return match.group(0)
+
+  @classmethod
+  def _make_mr_link(self, match):
+    try:
+      return '<a href="%s" target="_blank">%s</a>' % (reverse('jobbrowser.views.single_job', kwargs={'job': match.group(0)}), match.group(0))
+    except:
+      return match.group(0)
+
   def massaged_jobs_for_json(self, request, oozie_jobs, hue_jobs):
     jobs = []
     hue_jobs = dict([(script.dict.get('job_id'), script) for script in hue_jobs if script.dict.get('job_id')])

+ 2 - 2
apps/pig/src/pig/templates/app.mako

@@ -794,9 +794,9 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
               resizeLogs();
             }
             var _logsEl = $("#withLogs");
-            var newLines = data.logs.pig.split("\n").slice(_logsEl.text().split("\n").length);
+            var newLines = data.logs.pig.split("\n").slice(_logsEl.html().split("<br>").length);
             if (newLines.length > 0){
-              _logsEl.text(_logsEl.text() + newLines.join("\n") + "\n");
+              _logsEl.html(_logsEl.html() + newLines.join("<br>") + "<br>");
             }
             window.setTimeout(function () {
               resizeLogs();

+ 49 - 2
apps/pig/src/pig/tests.py

@@ -14,7 +14,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from liboozie.oozie_api_test import OozieServerProvider
 
 try:
   import json
@@ -29,9 +28,12 @@ from nose.tools import assert_true, assert_equal
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
-from pig.models import create_or_update_script, PigScript
+from liboozie.oozie_api_test import OozieServerProvider
 from oozie.tests import OozieBase
 
+from pig.models import create_or_update_script, PigScript
+from pig.api import OozieApi
+
 
 class TestPigBase(object):
   SCRIPT_ATTRS = {
@@ -52,6 +54,51 @@ class TestPigBase(object):
     attrs.update(TestPigBase.SCRIPT_ATTRS)
     return create_or_update_script(**attrs)
 
+  def make_log_links(self):
+    # FileBrowser
+    assert_equal(
+        """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a>  &lt;dir&gt;""",
+        OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp  <dir>')
+    )
+    assert_equal(
+        """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a>&lt;dir&gt;""",
+        OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp<dir>')
+    )
+    assert_equal(
+        """output: <a href="/filebrowser/view/user/romain/tmp" target="_blank">/user/romain/tmp</a>  &lt;dir&gt;""",
+        OozieApi._make_links('output: /user/romain/tmp  <dir>')
+    )
+    assert_equal(
+        'Successfully read 3760 records (112648 bytes) from: &quot;<a href="/filebrowser/view/user/hue/pig/examples/data/midsummer.txt" target="_blank">/user/hue/pig/examples/data/midsummer.txt</a>&quot;',
+        OozieApi._make_links('Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"')
+    )
+    assert_equal(
+        'data,upper_case  MAP_ONLY  <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>,',
+        OozieApi._make_links('data,upper_case  MAP_ONLY  hdfs://localhost:8020/user/romain/out/fffff,')
+    )
+    assert_equal(
+        'MAP_ONLY  <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013',
+        OozieApi._make_links('MAP_ONLY  hdfs://localhost:8020/user/romain/out/fffff\n2013')
+    )
+
+    # JobBrowser
+    assert_equal(
+        """<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+        OozieApi._make_links('job_201306261521_0058')
+    )
+    assert_equal(
+        """Hadoop Job IDs executed by Pig: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+        OozieApi._make_links('Hadoop Job IDs executed by Pig: job_201306261521_0058')
+    )
+    assert_equal(
+        """MapReduceLauncher  - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+        OozieApi._make_links('MapReduceLauncher  - HadoopJobId: job_201306261521_0058')
+    )
+    assert_equal(
+        """- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
+        OozieApi._make_links('- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058')
+    )
+
 
 class TestMock(TestPigBase):
 

+ 2 - 2
apps/pig/static/js/pig.ko.js

@@ -532,9 +532,9 @@ var PigViewModel = function (props) {
             $("#logsModal img").addClass("hide");
             $("#logsModal pre").removeClass("hide");
             var _logsEl = $("#logsModal pre");
-            var newLines = data.logs.pig.split("\n").slice(_logsEl.text().split("\n").length);
+            var newLines = data.logs.pig.split("\n").slice(_logsEl.html().split("<br>").length);
             if (newLines.length > 0){
-              _logsEl.text(_logsEl.text() + newLines.join("\n") + "\n");
+              _logsEl.html(_logsEl.html() + newLines.join("<br>") + "<br>");
             }
             if (self.showLogsAtEnd) {
               _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height());