Эх сурвалжийг харах

HUE-7803 [jb] Update through Hue to remove JT references in code path

Romain Rigaux 8 жил өмнө
parent
commit
c6be075

+ 47 - 0
apps/jobbrowser/src/jobbrowser/models.py

@@ -15,7 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
 import logging
+import re
 
 from django.core.urlresolvers import reverse
 from django.utils.html import escape
@@ -57,3 +59,48 @@ def can_kill_job(self, user):
     return True
 
   return user.username == self.user
+
+
+class LinkJobLogs(object):
+
+  @classmethod
+  def _make_hdfs_links(cls, log):
+    escaped_logs = escape(log)
+    return re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', LinkJobLogs._replace_hdfs_link, escaped_logs)
+
+  @classmethod
+  def _make_mr_links(cls, log):
+    escaped_logs = escape(log)
+    return re.sub('(job_[0-9]{12,}_[0-9]+)', LinkJobLogs._replace_mr_link, escaped_logs)
+
+  @classmethod
+  def _make_links(cls, log):
+    escaped_logs = escape(log)
+    hdfs_links = re.sub('((?<= |;)/|hdfs://)[^ <&\t;,\n]+', LinkJobLogs._replace_hdfs_link, escaped_logs)
+    return re.sub('(job_[0-9]{12,}_[0-9]+)', LinkJobLogs._replace_mr_link, hdfs_links)
+
+  @classmethod
+  def _replace_hdfs_link(self, match):
+    try:
+      return '<a href="%s">%s</a>' % (location_to_url(match.group(0), strict=False), match.group(0))
+    except:
+      LOG.exception('failed to replace hdfs links: %s' % (match.groups(),))
+      return match.group(0)
+
+  @classmethod
+  def _replace_mr_link(self, match):
+    try:
+      return '<a href="%s">%s</a>' % (reverse('jobbrowser.views.single_job', kwargs={'job': match.group(0)}), match.group(0))
+    except:
+      LOG.exception('failed to replace mr links: %s' % (match.groups(),))
+      return match.group(0)
+
+
+def format_unixtime_ms(unixtime):
+  """
+  Format a unix timestamp in ms to a human readable string
+  """
+  if unixtime:
+    return str(datetime.datetime.fromtimestamp(unixtime/1000).strftime("%x %X %Z"))
+  else:
+    return ""

+ 2 - 2
apps/jobbrowser/src/jobbrowser/views.py

@@ -38,8 +38,8 @@ from desktop.lib.json_utils import JSONEncoderForHTML
 from desktop.lib.exceptions import MessageException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.views import register_status_bar_view
+
 from hadoop import cluster
-from hadoop.api.jobtracker.ttypes import ThriftJobPriority, TaskTrackerNotFoundException, ThriftJobState
 from hadoop.yarn.clients import get_log_client
 from hadoop.yarn import resource_manager_api as resource_manager_api
 
@@ -55,7 +55,7 @@ except:
 
 from jobbrowser.conf import LOG_OFFSET, SHARE_JOBS
 from jobbrowser.api import get_api, ApplicationNotRunning, JobExpired
-from jobbrowser.models import Job, JobLinkage, Tracker, Cluster, can_view_job, LinkJobLogs, can_kill_job
+from jobbrowser.models import can_view_job, can_kill_job
 from jobbrowser.yarn_models import Application
 
 

+ 2 - 5
desktop/core/src/desktop/middleware.py

@@ -129,11 +129,8 @@ class ClusterMiddleware(object):
       if request.fs is not None:
         request.fs.setuser(request.user.username)
 
-      request.jt = cluster.get_default_mrcluster() # Deprecated, only there for MR1
-      if request.jt is not None:
-        request.jt.setuser(request.user.username)
-    else:
-      request.jt = None
+    # Deprecated
+    request.jt = None
 
 
 class NotificationMiddleware(object):

+ 2 - 0
desktop/libs/hadoop/src/hadoop/cluster.py

@@ -32,6 +32,8 @@ LOG = logging.getLogger(__name__)
 
 FS_CACHE = None
 FS_DEFAULT_NAME = 'default'
+MR_CACHE = None # MR now means YARN
+MR_NAME_CACHE = 'default'
 DEFAULT_USER = DEFAULT_USER.get()
 
 

+ 2 - 6
desktop/libs/liboozie/src/liboozie/submittion2_tests.py

@@ -207,8 +207,6 @@ class TestSubmission(OozieMockBase):
 
   def test_update_properties(self):
     finish = []
-    finish.append(MR_CLUSTERS.set_for_testing({'default': {}}))
-    finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     finish.append(YARN_CLUSTERS.set_for_testing({'default': {}}))
     finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     try:
@@ -228,13 +226,12 @@ class TestSubmission(OozieMockBase):
 
       clear_sys_caches()
       fs = cluster.get_hdfs()
-      jt = cluster.get_next_ha_mrcluster()[1]
       final_properties = properties.copy()
       final_properties.update({
         'jobTracker': 'jtaddress',
         'nameNode': fs.fs_defaultfs
       })
-      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
+      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       assert_equal(properties, submission.properties)
       submission._update_properties('jtaddress', 'deployment-directory')
       assert_equal(final_properties, submission.properties)
@@ -243,13 +240,12 @@ class TestSubmission(OozieMockBase):
       finish.append(MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker'))
       clear_sys_caches()
       fs = cluster.get_hdfs()
-      jt = cluster.get_next_ha_mrcluster()[1]
       final_properties = properties.copy()
       final_properties.update({
         'jobTracker': 'jobtracker',
         'nameNode': 'namenode'
       })
-      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
+      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       assert_equal(properties, submission.properties)
       submission._update_properties('jtaddress', 'deployment-directory')
       assert_equal(final_properties, submission.properties)

+ 2 - 6
desktop/libs/liboozie/src/liboozie/submittion_tests.py

@@ -166,8 +166,6 @@ class TestSubmission(OozieMockBase):
 
   def test_update_properties(self):
     finish = []
-    finish.append(MR_CLUSTERS.set_for_testing({'default': {}}))
-    finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     finish.append(YARN_CLUSTERS.set_for_testing({'default': {}}))
     finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     try:
@@ -186,13 +184,12 @@ class TestSubmission(OozieMockBase):
 
       clear_sys_caches()
       fs = cluster.get_hdfs()
-      jt = cluster.get_next_ha_mrcluster()[1]
       final_properties = properties.copy()
       final_properties.update({
         'jobTracker': 'jtaddress',
         'nameNode': fs.fs_defaultfs
       })
-      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
+      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       assert_equal(properties, submission.properties)
       submission._update_properties('jtaddress', 'deployment-directory')
       assert_equal(final_properties, submission.properties)
@@ -201,13 +198,12 @@ class TestSubmission(OozieMockBase):
       finish.append(MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker'))
       clear_sys_caches()
       fs = cluster.get_hdfs()
-      jt = cluster.get_next_ha_mrcluster()[1]
       final_properties = properties.copy()
       final_properties.update({
         'jobTracker': 'jobtracker',
         'nameNode': 'namenode'
       })
-      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
+      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       assert_equal(properties, submission.properties)
       submission._update_properties('jtaddress', 'deployment-directory')
       assert_equal(final_properties, submission.properties)