소스 검색

HUE-7142 [jb] Job log links to HDFS path opens on Hue 3

jdesjean 7 년 전
부모
커밋
41512b049b

+ 6 - 2
apps/filebrowser/src/filebrowser/views.py

@@ -1366,7 +1366,7 @@ def status(request):
     return render("status.mako", request, data)
 
 
-def location_to_url(location, strict=True):
+def location_to_url(location, strict=True, is_embeddable=False):
     """
     If possible, returns a file browser URL to the location.
     Prunes HDFS URI to path.
@@ -1384,7 +1384,11 @@ def location_to_url(location, strict=True):
     path = location
     if split_path[0] == 'hdfs':
       path = split_path[2]
-    return reverse("filebrowser.views.view", kwargs=dict(path=path))
+
+    filebrowser_path = reverse("filebrowser.views.view", kwargs=dict(path=path))
+    if is_embeddable and not filebrowser_path.startswith('/hue'):
+        filebrowser_path = '/hue' + filebrowser_path
+    return filebrowser_path
 
 
 def truncate(toTruncate, charsToKeep=50):

+ 1 - 0
apps/filebrowser/src/filebrowser/views_test.py

@@ -1145,6 +1145,7 @@ def test_location_to_url():
   prefix = '/filebrowser/view='
   assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('/var/lib/hadoop-hdfs', False))
   assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs'))
+  assert_equal('/hue' + prefix + '/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs', False, True))
   assert_equal(prefix + '/', location_to_url('hdfs://localhost:8020'))
   assert_equal(prefix + 's3a%3A//bucket/key', location_to_url('s3a://bucket/key'))
 

+ 1 - 1
apps/jobbrowser/src/jobbrowser/api2.py

@@ -106,7 +106,7 @@ def logs(request):
   app_type = json.loads(request.POST.get('type'))
   log_name = json.loads(request.POST.get('name'))
 
-  response['logs'] = get_api(request.user, interface).logs(app_id, app_type, log_name)
+  response['logs'] = get_api(request.user, interface).logs(app_id, app_type, log_name, json.loads(request.GET.get('is_embeddable', 'false').lower()))
   response['status'] = 0
 
   return JsonResponse(response)

+ 1 - 1
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -73,7 +73,7 @@ class Api(object):
 
   def action(self, app_ids, operation): return {}
 
-  def logs(self, appid, app_type, log_name): return {'progress': 0, 'logs': ''}
+  def logs(self, appid, app_type, log_name, is_embeddable=False): return {'progress': 0, 'logs': ''}
 
   def profile(self, appid, app_type, app_property, app_filters): return {} # Tasks, XML, counters...
 

+ 1 - 1
apps/jobbrowser/src/jobbrowser/apis/bundle_api.py

@@ -95,7 +95,7 @@ class BundleApi(Api):
     return _manage_oozie_job(self.user, action, app_ids)
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     request = MockDjangoRequest(self.user)
     data = get_oozie_job_log(request, job_id=appid)
 

+ 2 - 2
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -62,7 +62,7 @@ class DataEngClusterApi(Api):
     return {}
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     return {'logs': ''}
 
 
@@ -143,7 +143,7 @@ class DataEngJobApi(Api):
     return {}
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     return {'logs': ''}
 
 

+ 8 - 8
apps/jobbrowser/src/jobbrowser/apis/job_api.py

@@ -56,8 +56,8 @@ class JobApi(Api):
   def action(self, app_ids, operation):
     return self._get_api(app_ids).action(operation, app_ids)
 
-  def logs(self, appid, app_type, log_name):
-    return self._get_api(appid).logs(appid, app_type, log_name)
+  def logs(self, appid, app_type, log_name, is_embeddable=False):
+    return self._get_api(appid).logs(appid, app_type, log_name, is_embeddable)
 
   def profile(self, appid, app_type, app_property, app_filters):
     return self._get_api(appid).profile(appid, app_type, app_property, app_filters)
@@ -203,7 +203,7 @@ class YarnApi(Api):
       return {}
 
 
-  def logs(self, appid, app_type, log_name):
+  def logs(self, appid, app_type, log_name, is_embeddable=False):
     logs = ''
     try:
       if app_type == 'MAPREDUCE':
@@ -213,7 +213,7 @@ class YarnApi(Api):
           if logs and len(logs) == 4:
             logs = logs[1]
         else:
-          response = job_attempt_logs_json(MockDjangoRequest(self.user), job=appid, name=log_name)
+          response = job_attempt_logs_json(MockDjangoRequest(self.user), job=appid, name=log_name, is_embeddable=is_embeddable)
           logs = json.loads(response.content).get('log')
       elif app_type == 'SPARK':
         response = job_executor_logs(MockDjangoRequest(self.user), job=appid, name=log_name)
@@ -309,12 +309,12 @@ class YarnMapReduceTaskApi(Api):
     return common
 
 
-  def logs(self, appid, app_type, log_name):
+  def logs(self, appid, app_type, log_name, is_embeddable=False):
     if log_name == 'default':
       log_name = 'stdout'
 
     try:
-      response = job_attempt_logs_json(MockDjangoRequest(self.user), job=self.app_id, name=log_name)
+      response = job_attempt_logs_json(MockDjangoRequest(self.user), job=self.app_id, name=log_name, is_embeddable=is_embeddable)
       logs = json.loads(response.content)['log']
     except PopupException, e:
       LOG.warn('No task attempt found for default logs: %s' % e)
@@ -386,7 +386,7 @@ class YarnMapReduceTaskAttemptApi(Api):
     return common
 
 
-  def logs(self, appid, app_type, log_name):
+  def logs(self, appid, app_type, log_name, is_embeddable=False):
     if log_name == 'default':
       log_name = 'stdout'
 
@@ -487,7 +487,7 @@ class SparkExecutorApi(Api):
        "logs": executor['logs']
     }
 
-  def logs(self, appid, app_type, log_name, offset=LOG_OFFSET_BYTES):
+  def logs(self, appid, app_type, log_name, offset=LOG_OFFSET_BYTES, is_embeddable=False):
     log = ""
 
     if self._executors and self._executors[0]:

+ 2 - 2
apps/jobbrowser/src/jobbrowser/apis/livy_api.py

@@ -78,7 +78,7 @@ class LivySessionsApi(Api):
     return {}
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     return {'logs': ''}
 
 
@@ -153,7 +153,7 @@ class LivyJobApi(Api):
     return {}
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     return {'logs': ''}
 
 

+ 1 - 1
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -153,7 +153,7 @@ class QueryApi(Api):
     return message;
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     return {'logs': ''}
 
   def profile(self, appid, app_type, app_property, app_filters):

+ 1 - 1
apps/jobbrowser/src/jobbrowser/apis/schedule_api.py

@@ -97,7 +97,7 @@ class ScheduleApi(Api):
     return _manage_oozie_job(self.user, action, app_ids)
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     request = MockDjangoRequest(self.user)
     data = get_oozie_job_log(request, job_id=appid)
 

+ 1 - 1
apps/jobbrowser/src/jobbrowser/apis/workflow_api.py

@@ -101,7 +101,7 @@ class WorkflowApi(Api):
     return _manage_oozie_job(self.user, action, app_ids)
 
 
-  def logs(self, appid, app_type, log_name=None):
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     if '@' in appid:
       return WorkflowActionApi(self.user).logs(appid, app_type)
 

+ 7 - 6
apps/jobbrowser/src/jobbrowser/models.py

@@ -17,6 +17,7 @@
 
 import datetime
 import logging
+import functools
 import re
 
 from django.core.urlresolvers import reverse
@@ -64,9 +65,9 @@ def can_kill_job(self, user):
 class LinkJobLogs(object):
 
   @classmethod
-  def _make_hdfs_links(cls, log):
+  def _make_hdfs_links(cls, log, is_embeddable=False):
     escaped_logs = escape(log)
-    return re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', LinkJobLogs._replace_hdfs_link, escaped_logs)
+    return re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', functools.partial(LinkJobLogs._replace_hdfs_link, is_embeddable), escaped_logs)
 
   @classmethod
   def _make_mr_links(cls, log):
@@ -74,15 +75,15 @@ class LinkJobLogs(object):
     return re.sub('(job_[0-9]{12,}_[0-9]+)', LinkJobLogs._replace_mr_link, escaped_logs)
 
   @classmethod
-  def _make_links(cls, log):
+  def _make_links(cls, log, is_embeddable=False):
     escaped_logs = escape(log)
-    hdfs_links = re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', LinkJobLogs._replace_hdfs_link, escaped_logs)
+    hdfs_links = re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', functools.partial(LinkJobLogs._replace_hdfs_link, is_embeddable), escaped_logs)
     return re.sub('(job_[0-9]{12,}_[0-9]+)', LinkJobLogs._replace_mr_link, hdfs_links)
 
   @classmethod
-  def _replace_hdfs_link(self, match):
+  def _replace_hdfs_link(self, is_embeddable=False, match=None):
     try:
-      return '<a href="%s">%s</a>' % (location_to_url(match.group(0), strict=False), match.group(0))
+      return '<a href="%s">%s</a>' % (location_to_url(match.group(0), strict=False, is_embeddable=is_embeddable), match.group(0))
     except:
       LOG.exception('failed to replace hdfs links: %s' % (match.groups(),))
       return match.group(0)

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -2116,7 +2116,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
       self.fetchLogs = function (name) {
         self.logs('');
-        $.post("/jobbrowser/api/job/logs", {
+        $.post("/jobbrowser/api/job/logs?is_embeddable=${ str(is_embeddable).lower() }", {
           app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface),
           type: ko.mapping.toJSON(self.type),

+ 2 - 2
apps/jobbrowser/src/jobbrowser/views.py

@@ -326,7 +326,7 @@ def job_attempt_logs(request, job, attempt_index=0):
 
 
 @check_job_permission
-def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=LOG_OFFSET_BYTES):
+def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=LOG_OFFSET_BYTES, is_embeddable=False):
   """For async log retrieval as Yarn servers are very slow"""
   log_link = None
   response = {'status': -1}
@@ -369,7 +369,7 @@ def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=L
       log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
 
       response['status'] = 0
-      response['log'] = LinkJobLogs._make_hdfs_links(log)
+      response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable)
     except Exception, e:
       response['log'] = _('Failed to retrieve log: %s' % e)
       try: