浏览代码

HUE-1176 [jb] List tasks of a MR2 Job

Romain Rigaux 9 年之前
父节点
当前提交
ce291c5

+ 18 - 8
apps/jobbrowser/src/jobbrowser/api2.py

@@ -62,16 +62,16 @@ def job(request):
   response = {'status': -1}
   response = {'status': -1}
 
 
   interface = json.loads(request.POST.get('interface'))
   interface = json.loads(request.POST.get('interface'))
-  appid = json.loads(request.POST.get('appid'))
+  app_id = json.loads(request.POST.get('app_id'))
 
 
-  response['app'] = get_api(request.user, interface).app(appid)
+  response['app'] = get_api(request.user, interface).app(app_id)
   response['status'] = 0
   response['status'] = 0
 
 
   return JsonResponse(response)
   return JsonResponse(response)
 
 
 
 
 @api_error_handler
 @api_error_handler
-def kill(request): return {}
+def action(request): return {}
 
 
 
 
 @api_error_handler
 @api_error_handler
@@ -79,18 +79,28 @@ def logs(request):
   response = {'status': -1}
   response = {'status': -1}
 
 
   interface = json.loads(request.POST.get('interface'))
   interface = json.loads(request.POST.get('interface'))
-  appid = json.loads(request.POST.get('appid'))
+  app_id = json.loads(request.POST.get('app_id'))
   app_type = json.loads(request.POST.get('type'))
   app_type = json.loads(request.POST.get('type'))
 
 
-  response['logs'] = get_api(request.user, interface).logs(appid, app_type)
+  response['logs'] = get_api(request.user, interface).logs(app_id, app_type)
   response['status'] = 0
   response['status'] = 0
 
 
   return JsonResponse(response)
   return JsonResponse(response)
 
 
 
 
 @api_error_handler
 @api_error_handler
-def tasks(request): return []
+def profile(request):
+  response = {'status': -1}
+
+  interface = json.loads(request.POST.get('interface'))
+  app_id = json.loads(request.POST.get('app_id'))
+  app_type = json.loads(request.POST.get('app_type'))
+  app_property = json.loads(request.POST.get('app_property'))
 
 
+  api = get_api(request.user, interface)
+  api._set_request(request) # For YARN
 
 
-@api_error_handler
-def profile(self, appid, app_type, property): return {}
+  response[app_property] = api.profile(app_id, app_type, app_property)
+  response['status'] = 0
+
+  return JsonResponse(response)

+ 6 - 2
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -40,10 +40,11 @@ def get_api(user, interface):
     raise PopupException(_('Interface %s is unknown') % interface)
     raise PopupException(_('Interface %s is unknown') % interface)
 
 
 
 
-class Api():
+class Api(object):
 
 
   def __init__(self, user):
   def __init__(self, user):
     self.user = user
     self.user = user
+    self.request = None
 
 
   def apps(self): return []
   def apps(self): return []
 
 
@@ -55,7 +56,10 @@ class Api():
 
 
   def logs(self, appid, app_type): return {'progress': 0, 'logs': {'default': ''}}
   def logs(self, appid, app_type): return {'progress': 0, 'logs': {'default': ''}}
 
 
-  def profile(self, appid, app_type, property): return {} # Tasks, XML, counters...
+  def profile(self, appid, app_type, app_property): return {} # Tasks, XML, counters...
+
+  def _set_request(self, request):
+    self.request = request
 
 
 
 
 class MockDjangoRequest():
 class MockDjangoRequest():

+ 15 - 8
apps/jobbrowser/src/jobbrowser/apis/job_api.py

@@ -37,8 +37,9 @@ class JobApi(Api):
 
 
   def __init__(self, user):
   def __init__(self, user):
     self.user =  user
     self.user =  user
-    self.yarn_api = YarnApi(user)
+    self.yarn_api = YarnApi(user) # TODO: actually long term move job aggregations to the frontend instead probably
     self.impala_api = ImpalaApi(user)
     self.impala_api = ImpalaApi(user)
+    self.request = None
 
 
   def apps(self):
   def apps(self):
     jobs = self.self.yarn_api.apps()
     jobs = self.self.yarn_api.apps()
@@ -52,14 +53,13 @@ class JobApi(Api):
   def logs(self, appid):
   def logs(self, appid):
     return self._get_api(appid).logs(appid)
     return self._get_api(appid).logs(appid)
 
 
-  def profile(self, appid, app_type, property):
-    return self._get_api(appid).logs(appid, app_type, property)
+  def profile(self, appid, app_type, app_property):
+    return self._get_api(appid).profile(appid, app_type, app_property)
 
 
   def _get_api(self, appid):
   def _get_api(self, appid):
     return self.impala_api if not appid.startswith('application_') else self.yarn_api
     return self.impala_api if not appid.startswith('application_') else self.yarn_api
 
 
 
 
-
 class YarnApi(Api):
 class YarnApi(Api):
   """YARN, MR, Spark"""
   """YARN, MR, Spark"""
 
 
@@ -76,6 +76,7 @@ class YarnApi(Api):
         'submitted': 10 * 3600
         'submitted': 10 * 3600
     } for app in jobs]
     } for app in jobs]
 
 
+
   def app(self, appid):
   def app(self, appid):
     app = NativeYarnApi(self.user).get_job(jobid=appid)
     app = NativeYarnApi(self.user).get_job(jobid=appid)
 
 
@@ -102,9 +103,11 @@ class YarnApi(Api):
           'finishedReduces': app.finishedReduces,
           'finishedReduces': app.finishedReduces,
           'desiredMaps': app.desiredMaps,
           'desiredMaps': app.desiredMaps,
           'desiredReduces': app.desiredReduces,
           'desiredReduces': app.desiredReduces,
+          'tasks': []
       }
       }
     return common
     return common
 
 
+
   def logs(self, appid, app_type):
   def logs(self, appid, app_type):
     if app_type == 'MAPREDUCE':
     if app_type == 'MAPREDUCE':
       response = job_attempt_logs_json(MockDjangoRequest(self.user), job=appid)
       response = job_attempt_logs_json(MockDjangoRequest(self.user), job=appid)
@@ -114,15 +117,19 @@ class YarnApi(Api):
     return {'progress': 0, 'logs': {'default': logs}}
     return {'progress': 0, 'logs': {'default': logs}}
 
 
 
 
-  def profile(self, appid, app_type, property):
+  def profile(self, appid, app_type, app_property):
     if app_type == 'MAPREDUCE':
     if app_type == 'MAPREDUCE':
-      if property == 'tasks':
-        response = tasks(MockDjangoRequest(self.user), job=appid)
-        print response.context['task_list']
+      if app_property == 'tasks':
+        response = tasks(self.request, job=appid)
+        return json.loads(response.content)
 
 
     return {}
     return {}
 
 
 
 
+  def _set_request(self, request):
+    self.request = request
+
+
 class YarnAtsApi(Api):
 class YarnAtsApi(Api):
   pass
   pass
 
 

+ 30 - 10
apps/jobbrowser/src/jobbrowser/templates/apps.mako

@@ -304,7 +304,27 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
     </div>
     </div>
 
 
     <div class="tab-pane" id="job-mapreduce-page-tasks">
     <div class="tab-pane" id="job-mapreduce-page-tasks">
+      <a href="javascript:void(0)" data-bind="click: function() { return fetchProfile('tasks'); }">Load</a>
+
       <table class="table table-condensed">
       <table class="table table-condensed">
+        <thead>
+        <tr>
+          <th width="1%"><div class="select-all hueCheckbox fa"></div></th>
+          <th>${_('Type')}</th>
+          <th>${_('Id')}</th>
+          <th>${_('Logs')}</th>
+          <th>${_('Url')}</th>
+        </tr>
+        </thead>
+        <tbody data-bind="foreach: properties['tasks']()['task_list']">
+          <tr data-bind="click: function() { $root.job().fetchJob(id); }">
+            <td><div class="hueCheckbox fa"></div></td>
+            <td data-bind="text: type"></td>
+            <td data-bind="text: id"></td>
+            <td data-bind="text: logs"></td>
+            <td data-bind="text: url"></td>
+          </tr>
+        </tbody>
       </table>
       </table>
     </div>
     </div>
 
 
@@ -394,7 +414,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
 
   <div class="tab-content">
   <div class="tab-content">
     <div class="tab-pane active" id="workflow-page-graph">
     <div class="tab-pane active" id="workflow-page-graph">
-      <span data-bind="html: properties().graph"></span>
+      <span data-bind="html: properties['graph']"></span>
     </div>
     </div>
 
 
     <div class="tab-pane" id="workflow-page-logs">
     <div class="tab-pane" id="workflow-page-logs">
@@ -477,7 +497,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
 
       self.coordVM = new RunningCoordinatorModel([]);
       self.coordVM = new RunningCoordinatorModel([]);
 
 
-      self.properties = ko.observableDefault(job.properties, {});
+      self.properties = ko.mapping.fromJS(job.properties || {});
       self.mainType = ko.observable(vm.interface());
       self.mainType = ko.observable(vm.interface());
 
 
       self.loadingJob = ko.observable(false);
       self.loadingJob = ko.observable(false);
@@ -488,7 +508,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         vm.breadcrumbs.push({'id': self.id(), 'name': self.name(), 'type': self.type()});
         vm.breadcrumbs.push({'id': self.id(), 'name': self.name(), 'type': self.type()});
 
 
         $.post("/jobbrowser/api/job", {
         $.post("/jobbrowser/api/job", {
-          appid: ko.mapping.toJSON(self.id),
+          app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface)
           interface: ko.mapping.toJSON(vm.interface)
         }, function (data) {
         }, function (data) {
           if (data.status == 0) {
           if (data.status == 0) {
@@ -507,7 +527,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
 
       self.fetchLogs = function (name) {
       self.fetchLogs = function (name) {
         $.post("/jobbrowser/api/job/logs", {
         $.post("/jobbrowser/api/job/logs", {
-          appid: ko.mapping.toJSON(self.id),
+          app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface),
           interface: ko.mapping.toJSON(vm.interface),
           type: ko.mapping.toJSON(self.type),
           type: ko.mapping.toJSON(self.type),
           name: name ? name : 'default'
           name: name ? name : 'default'
@@ -523,13 +543,13 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
 
       self.fetchProfile = function (name) {
       self.fetchProfile = function (name) {
         $.post("/jobbrowser/api/job/profile", {
         $.post("/jobbrowser/api/job/profile", {
-          appid: ko.mapping.toJSON(self.id),
+          app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface),
           interface: ko.mapping.toJSON(vm.interface),
-          type: ko.mapping.toJSON(self.type),
-          name: name
+          app_type: ko.mapping.toJSON(self.type),
+          app_property: ko.mapping.toJSON(name)
         }, function (data) {
         }, function (data) {
           if (data.status == 0) {
           if (data.status == 0) {
-            self.properties['default'](data.logs.logs['default']);
+            self.properties[name](data[name]);
           } else {
           } else {
             $(document).trigger("error", data.message);
             $(document).trigger("error", data.message);
           }
           }
@@ -581,7 +601,6 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       self.jobs = new Jobs(self, options);
       self.jobs = new Jobs(self, options);
       self.job = ko.observable();
       self.job = ko.observable();
 
 
-      //self.section = ko.observable('apps');
       self.interface = ko.observable('apps');
       self.interface = ko.observable('apps');
       self.interface.subscribe(function (val) {
       self.interface.subscribe(function (val) {
         hueUtils.changeURL('#!' + val);
         hueUtils.changeURL('#!' + val);
@@ -616,9 +635,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           h = h.substr(2);
           h = h.substr(2);
         }
         }
         switch (h) {
         switch (h) {
-          case 'schedules':
           case 'apps':
           case 'apps':
           case 'workflows':
           case 'workflows':
+          case 'schedules':
             viewModel.interface(h);
             viewModel.interface(h);
             break;
             break;
           default:
           default:
@@ -635,6 +654,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
   })();
   })();
 </script>
 </script>
 </span>
 </span>
+
 % if not is_embeddable:
 % if not is_embeddable:
 ${ commonfooter(request, messages) | n,unicode }
 ${ commonfooter(request, messages) | n,unicode }
 % endif
 % endif

+ 1 - 0
apps/jobbrowser/src/jobbrowser/urls.py

@@ -58,5 +58,6 @@ urlpatterns += patterns('jobbrowser.api2',
   url(r'api/jobs', 'jobs', name='jobs'),
   url(r'api/jobs', 'jobs', name='jobs'),
   url(r'api/job/logs', 'logs', name='logs'),
   url(r'api/job/logs', 'logs', name='logs'),
   url(r'api/job/profile', 'profile', name='profile'),
   url(r'api/job/profile', 'profile', name='profile'),
+  url(r'api/job/action', 'action', name='action'),
   url(r'api/job', 'job', name='job'),
   url(r'api/job', 'job', name='job'),
 )
 )

+ 9 - 0
apps/jobbrowser/src/jobbrowser/views.py

@@ -425,6 +425,15 @@ def tasks(request, job):
 
 
   filter_params = copy_query_dict(request.GET, ('tasktype', 'taskstate', 'tasktext')).urlencode()
   filter_params = copy_query_dict(request.GET, ('tasktype', 'taskstate', 'tasktext')).urlencode()
 
 
+  if request.is_ajax():
+    return JsonResponse({
+      'filter_params': filter_params,
+      'task_list': [massage_task_for_json(task) for task in task_list],
+      'tasktype': ttypes,
+      'taskstate': tstates,
+      'tasktext': ttext
+    })
+
   return render("tasks.mako", request, {
   return render("tasks.mako", request, {
     'request': request,
     'request': request,
     'filter_params': filter_params,
     'filter_params': filter_params,