Эх сурвалжийг харах

HUE-1625 [jb] Make resource manager API dynamic

Romain Rigaux 12 жил өмнө
parent
commit
f17718e

+ 3 - 3
apps/jobbrowser/src/jobbrowser/api.py

@@ -190,7 +190,6 @@ class YarnApi(JobBrowserApi):
     self.user = user
     self.resource_manager_api = resource_manager_api.get_resource_manager()
     self.mapreduce_api = mapreduce_api.get_mapreduce_api()
-    self.node_manager_api = node_manager_api.get_resource_manager_api()
     self.history_server_api = history_server_api.get_history_server_api()
 
   def get_job_link(self, job_id):
@@ -255,5 +254,6 @@ class YarnApi(JobBrowserApi):
   def get_task(self, jobid, task_id):
     return self.get_job(jobid).task(task_id)
 
-  def get_tracker(self, container_id):
-    return Container(self.node_manager_api.container(container_id))
+  def get_tracker(self, node_manager_http_address, container_id):
+    api = node_manager_api.get_resource_manager_api('http://' + node_manager_http_address)
+    return Container(api.container(container_id))

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/attempt.mako

@@ -117,7 +117,7 @@ ${ comps.menubar() }
                   <td>${_('Task Tracker')}</td>
                   <td>
                       % if attempt.is_mr2:
-                      ${ comps.get_container_link(status, attempt.taskTrackerId) }
+                        ${ comps.get_container_link(status, attempt.nodeHttpAddress, attempt.taskTrackerId) }
                       % else:
                         <a href="/jobbrowser/trackers/${attempt.taskTrackerId}"
                            class="task_tracker_link">${attempt.taskTrackerId}</a>

+ 67 - 0
apps/jobbrowser/src/jobbrowser/templates/container.mako

@@ -0,0 +1,67 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+<%!
+  from desktop.views import commonheader, commonfooter
+  from django.utils.translation import ugettext as _
+%>
+
+<%namespace name="comps" file="jobbrowser_components.mako" />
+
+${ commonheader(_('Container: %(trackerId)s') % dict(trackerId=tracker.trackerId), "jobbrowser", user) | n,unicode }
+${ comps.menubar()}
+
+<div class="container-fluid">
+  <div class="row-fluid">
+    <div class="span12">
+      <div class="card card-home">
+        <div class="card-body">
+          <p>
+
+    <h1>${_('Container at %(trackerHost)s on port %(trackerPort)s') % dict(trackerHost=tracker.host, trackerPort=tracker.httpPort)}</h1>
+    <div>
+        <dl>
+            <dt>${_('ID')}</dt>
+            <dd>${ tracker.trackerId }</dd>
+        </dl>
+    </div>
+
+    <h2>${_('Memory Metrics')}</h2>
+    <div>
+        <dl>
+            <dt>${_('Node Id')}</dt>
+            <dd>${tracker.nodeId }</dd>
+            <dt>${_('State')}</dt>
+            <dd>${tracker.state }</dd>
+            <dt>${_('User')}</dt>
+            <dd>${tracker.user}</dd>
+            <dt>${_('Diagnostics')}</dt>
+            <dd>${tracker.diagnostics}</dd>
+            <dt>${_('Total Memory Needed in MB')}</dt>
+            <dd>${tracker.totalMemoryNeededMB}</dd>
+            <dt>${_('Exit Code')}</dt>
+            <dd>${tracker.exitCode}</dd>
+        </dl>
+    </div>
+
+          <a class="btn" href="javascript:history.back()">${_('Back')}</a>
+
+          </p>
+        </div>
+      </div>
+    </div>
+</div>
+
+${ commonfooter(messages) | n,unicode }

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/job.mako

@@ -186,7 +186,7 @@ ${ comps.menubar() }
                           </a>
                         </td>
                         <td>${ attempt['id'] }</td>
-                        <td>${ comps.get_container_link(job.status, attempt['containerId']) }</td>
+                        <td>${ comps.get_container_link(job.status, attempt['nodeHttpAddress'], attempt['containerId']) }</td>
                       </tr>
                       % endfor
                     </tbody>

+ 2 - 2
apps/jobbrowser/src/jobbrowser/templates/jobbrowser_components.mako

@@ -191,10 +191,10 @@
     % endif
 </%def>
 
-<%def name="get_container_link(status, container_id)">
+<%def name="get_container_link(status, node_manager_http_address, container_id)">
     ## As soon as the job finishes the container disappears
     % if status.lower() in ('running', 'prep', 'accepted', 'finishing'):
-        <a href="${ url('jobbrowser.views.single_tracker',trackerid=container_id) }" class="task_tracker_link">${ container_id }</a>
+        <a href="${ url('jobbrowser.views.container', node_manager_http_address=node_manager_http_address, containerid=container_id) }" class="task_tracker_link">${ container_id }</a>
     % else:
         ${ container_id }
     % endif

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/task.mako

@@ -85,7 +85,7 @@ ${ comps.menubar() }
                       <td><span class="status_link ${attempt.state}">${attempt.state}</span></td>
                       <td>
                         % if task.is_mr2:
-                          ${ comps.get_container_link(task.state, attempt.taskTrackerId) }
+                          ${ comps.get_container_link(task.state, attempt.nodeHttpAddress, attempt.taskTrackerId) }
                         % else:
                           <a href="/jobbrowser/trackers/${attempt.taskTrackerId}" class="task_tracker_link">${attempt.taskTrackerId}</a>
                         % endif

+ 0 - 22
apps/jobbrowser/src/jobbrowser/templates/tasktracker.mako

@@ -35,32 +35,11 @@ ${ comps.menubar()}
         <dl>
             <dt>${_('ID')}</dt>
             <dd>${ tracker.trackerId }</dd>
-            % if not tracker.is_mr2:
             <dt>${_('Last heard from at')}</dt>
             <dd>${ tracker.lastSeenFormatted }.</dd>
-            % endif
         </dl>
     </div>
 
-    % if tracker.is_mr2:
-    <h2>${_('Memory Metrics')}</h2>
-    <div>
-        <dl>
-            <dt>${_('Node Id')}</dt>
-            <dd>${tracker.nodeId }</dd>
-            <dt>${_('State')}</dt>
-            <dd>${tracker.state }</dd>
-            <dt>${_('User')}</dt>
-            <dd>${tracker.user}</dd>
-            <dt>${_('Diagnostics')}</dt>
-            <dd>${tracker.diagnostics}</dd>
-            <dt>${_('Total Memory Needed in MB')}</dt>
-            <dd>${tracker.totalMemoryNeededMB}</dd>
-            <dt>${_('Exit Code')}</dt>
-            <dd>${tracker.exitCode}</dd>
-        </dl>
-    </div>
-    % else:
     <h2>${_('Memory Metrics')}</h2>
     <div>
         <dl>
@@ -86,7 +65,6 @@ ${ comps.menubar()}
             <dd>${tracker.maxReduceTasks}</dd>
         </dl>
     </div>
-    % endif
 
           <a class="btn" href="javascript:history.back()">${_('Back')}</a>
 

+ 0 - 1
apps/jobbrowser/src/jobbrowser/tests.py

@@ -30,7 +30,6 @@ from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_to_group
 from desktop.models import Document
 from hadoop import cluster
-from hadoop import pseudo_hdfs4
 from hadoop.conf import YARN_CLUSTERS
 from hadoop.yarn import resource_manager_api, mapreduce_api, history_server_api
 from liboozie.oozie_api_test import OozieServerProvider

+ 1 - 0
apps/jobbrowser/src/jobbrowser/urls.py

@@ -32,6 +32,7 @@ urlpatterns = patterns('jobbrowser.views',
   url(r'^jobs/(?P<job>\w+)/tasks/(?P<taskid>\w+)/attempts/(?P<attemptid>\w+)/logs$', 'single_task_attempt_logs', name='single_task_attempt_logs'),
   url(r'^jobs/(\w+)/tasks/(\w+)/attempts/(?P<attemptid>\w+)/kill$', 'kill_task_attempt', name='kill_task_attempt'),
   url(r'^trackers/(?P<trackerid>.+)$', 'single_tracker', name='single_tracker'),
+  url(r'^container/(?P<node_manager_http_address>.+)/(?P<containerid>.+)$', 'container', name='container'),
 
   # MR2 specific
   url(r'^jobs/(?P<job>\w+)/job_attempt_logs/(?P<attempt_index>\d+)$', 'job_attempt_logs', name='job_attempt_logs'),

+ 12 - 1
apps/jobbrowser/src/jobbrowser/views.py

@@ -415,9 +415,20 @@ def single_tracker(request, trackerid):
   try:
     tracker = jt.get_tracker(trackerid)
   except Exception, e:
-    raise PopupException(_('The container disappears as soon as the job finishes.'), detail=e)
+    raise PopupException(_('The tracker could not be contacted.'), detail=e)
   return render("tasktracker.mako", request, {'tracker':tracker})
 
+def container(request, node_manager_http_address, containerid):
+  jt = get_api(request.user, request.jt)
+
+  try:
+    tracker = jt.get_tracker(node_manager_http_address, containerid)
+  except Exception, e:
+    # TODO: add a redirect of some kind
+    raise PopupException(_('The container disappears as soon as the job finishes.'), detail=e)
+  return render("container.mako", request, {'tracker':tracker})
+
+
 def clusterstatus(request):
   """
   We get here from /clusterstatus

+ 0 - 3
desktop/conf.dist/hue.ini

@@ -354,9 +354,6 @@
       # URL of the HistoryServer API
       # history_server_api_url=http://localhost:19888
 
-      # URL of the NodeManager API
-      # node_manager_api_url=http://localhost:8042
-
   # Configuration for MapReduce (MR1)
   # ------------------------------------------------------------------------
   [[mapred_clusters]]

+ 0 - 3
desktop/conf/pseudo-distributed.ini.tmpl

@@ -359,9 +359,6 @@
       # URL of the HistoryServer API
       # history_server_api_url=http://localhost:19888
 
-      # URL of the NodeManager API
-      # node_manager_api_url=http://localhost:8042
-
   # Configuration for MapReduce (MR1)
   # ------------------------------------------------------------------------
   [[mapred_clusters]]

+ 0 - 3
desktop/libs/hadoop/src/hadoop/conf.py

@@ -219,9 +219,6 @@ YARN_CLUSTERS = UnspecifiedConfigSection(
       HISTORY_SERVER_API_URL=Config("history_server_api_url",
                   default='http://localhost:19888',
                   help="URL of the HistoryServer API"),
-      NODE_MANAGER_API_URL=Config("node_manager_api_url",
-                  default='http://localhost:8042',
-                  help="URL of the HistoryServer API"),
     )
   )
 )

+ 0 - 1
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -522,7 +522,6 @@ def shared_cluster():
       hadoop.conf.YARN_CLUSTERS['default'].PORT.set_for_testing(cluster._rm_port),
 
       hadoop.conf.YARN_CLUSTERS['default'].RESOURCE_MANAGER_API_URL.set_for_testing('http://%s:%s' % (cluster._fqdn, cluster._rm_webapp_port,)),
-      hadoop.conf.YARN_CLUSTERS['default'].NODE_MANAGER_API_URL.set_for_testing('http://%s:%s' % (cluster._fqdn, cluster._nm_webapp_port,)),
       hadoop.conf.YARN_CLUSTERS['default'].PROXY_API_URL.set_for_testing('http://%s:%s' % (cluster._fqdn, cluster._rm_webapp_port,)),
       hadoop.conf.YARN_CLUSTERS['default'].HISTORY_SERVER_API_URL.set_for_testing('%s:%s' % (cluster._fqdn, cluster._jh_web_port,)),
     ]

+ 2 - 15
desktop/libs/hadoop/src/hadoop/yarn/node_manager_api.py

@@ -17,11 +17,9 @@
 
 import logging
 import posixpath
-import threading
 
 from desktop.lib.rest.http_client import HttpClient
 from desktop.lib.rest.resource import Resource
-from hadoop import cluster
 
 
 LOG = logging.getLogger(__name__)
@@ -30,21 +28,10 @@ DEFAULT_USER = 'hue'
 _API_VERSION = 'v1'
 _JSON_CONTENT_TYPE = 'application/json'
 
-_api_cache = None
-_api_cache_lock = threading.Lock()
 
 
-def get_resource_manager_api():
-  global _api_cache
-  if _api_cache is None:
-    _api_cache_lock.acquire()
-    try:
-      if _api_cache is None:
-        yarn_cluster = cluster.get_cluster_conf_for_job_submission()
-        _api_cache = ResourceManagerApi(yarn_cluster.NODE_MANAGER_API_URL.get())
-    finally:
-      _api_cache_lock.release()
-  return _api_cache
+def get_resource_manager_api(api_url):
+  return ResourceManagerApi(api_url)
 
 
 class ResourceManagerApi(object):