瀏覽代碼

HUE-1176 [jb] Basic skeleton to fetch MR job logs

Romain Rigaux 9 年之前
父節點
當前提交
118b94a8c7

+ 9 - 4
apps/jobbrowser/src/jobbrowser/api2.py

@@ -75,15 +75,20 @@ def kill(request): return {}
 
 
 @api_error_handler
-def progress(request): return {'progress': 0}
+def logs(request):
+  response = {'status': -1}
 
+  interface = json.loads(request.POST.get('interface'))
+  appid = json.loads(request.POST.get('appid'))
 
-@api_error_handler
-def tasks(request): return []
+  response['logs'] = get_api(request.user, interface).logs(appid)
+  response['status'] = 0
+
+  return JsonResponse(response)
 
 
 @api_error_handler
-def logs(request): return {'stderr': '', 'stdout': ''}
+def tasks(request): return []
 
 
 @api_error_handler

+ 8 - 12
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -26,16 +26,16 @@ LOG = logging.getLogger(__name__)
 
 
 def get_api(user, interface):
-  from jobbrowser.apis.batch_api import BatchApi
+  from jobbrowser.apis.workflow_api import WorkflowApi
   from jobbrowser.apis.job_api import YarnApi
   from jobbrowser.apis.schedule_api import ScheduleApi
 
-  if interface == 'batches':
-    return BatchApi(user)
+  if interface == 'apps':
+    return YarnApi(user)
+  elif interface == 'workflows':
+    return WorkflowApi(user)
   elif interface == 'schedules':
     return ScheduleApi(user)
-  elif interface == 'apps':
-    return YarnApi(user)
   else:
     raise PopupException(_('Interface %s is unknown') % interface)
 
@@ -49,12 +49,8 @@ class Api():
 
   def app(self, appid): return {}
 
-  def kill(self): return {}
-
-  def progress(self): return {'progress': 0}
-
-  def tasks(self): return []
+  def action(self, appid): return {} # Kill, suspend...
 
-  def logs(self): return {'stderr': '', 'stdout': ''}
+  def logs(self, appid): return {'progress': 0, 'logs': {'default': ''}}
 
-  def profile(self): return {}
+  def profile(self, appid): return {} # Tasks, XML, counters...

+ 19 - 1
apps/jobbrowser/src/jobbrowser/apis/job_api.py

@@ -15,11 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import logging
 
 from django.utils.translation import ugettext as _
 
 from jobbrowser.apis.base_api import Api
+from jobbrowser.views import job_attempt_logs_json
 
 
 LOG = logging.getLogger(__name__)
@@ -47,6 +49,9 @@ class JobApi(Api):
   def app(self, appid):
     return self._get_api(appid).app(appid)
 
+  def logs(self, appid):
+    return self._get_api(appid).logs(appid)
+
   def _get_api(self, appid):
     return self.impala_api if not appid.startswith('application_') else self.yarn_api
 
@@ -69,7 +74,7 @@ class YarnApi(Api):
 
   def app(self, appid):
     app = NativeYarnApi(self.user).get_job(jobid=appid)
-    print app.finishedMaps
+
     return {
         'id': app.jobId,
         'name': app.name,
@@ -81,6 +86,19 @@ class YarnApi(Api):
         'submitted': 10 * 3600
     }
 
+  def logs(self, appid):
+    # name = 'stdout'
+    # attempt_index
+    # offset=log_offset
+    data = job_attempt_logs_json(YarnRequest(self.user), job=appid)
+    return {'progress': 0, 'logs': {'default': json.loads(data.content)['log']}}
+
+
+class YarnRequest():
+  def __init__(self, user):
+    self.user = user
+    self.jt = None
+
 
 class YarnAtsApi(Api):
   pass

+ 4 - 1
apps/jobbrowser/src/jobbrowser/apis/batch_api.py → apps/jobbrowser/src/jobbrowser/apis/workflow_api.py

@@ -32,7 +32,7 @@ except Exception, e:
   LOG.exception('Some application are not enabled for Job Browser v2: %s' % e)
 
 
-class BatchApi(Api):
+class WorkflowApi(Api):
 
   def apps(self):
     oozie_api = get_oozie(self.user)
@@ -55,3 +55,6 @@ class BatchApi(Api):
     workflow = oozie_api.get_job(jobid=appid)
 
     return {'id': workflow.id, 'name': workflow.appName, 'status': workflow.status}
+
+  def progress(self, appid):
+    pass

+ 70 - 13
apps/jobbrowser/src/jobbrowser/templates/apps.mako

@@ -75,7 +75,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           </ul>
             <span class="form-inline">
               <input class="btn btn-status" type="radio" name="interface" value="apps" data-bind="checked: interface" id="apps"><label for="apps">${ _('Apps') }</label>
-              <input class="btn btn-status" type="radio" name="interface" value="batches" data-bind="checked: interface" id="batches"><label for="batches">${ _('Workflows') }</label>
+              <input class="btn btn-status" type="radio" name="interface" value="workflows" data-bind="checked: interface" id="workflows"><label for="workflows">${ _('Workflows') }</label>
               <input class="btn btn-status" type="radio" name="interface" value="schedules" data-bind="checked: interface" id="schedules"><label for="schedules">${ _('Schedules') }</label>
               <input class="btn btn-status" type="radio" name="interface" value="bundles" data-bind="checked: interface" id="bundles"><label for="bundles">${ _('Bundles') }</label>
             </span>
@@ -174,8 +174,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
               <div data-bind="template: { name: 'job-page', data: $root.job() }"></div>
             <!-- /ko -->
 
-            <!-- ko if: $root.job().mainType() == 'batches' -->
-              <div data-bind="template: { name: 'batch-page', data: $root.job() }"></div>
+            <!-- ko if: $root.job().mainType() == 'workflows' -->
+              <div data-bind="template: { name: 'workflow-page', data: $root.job() }"></div>
             <!-- /ko -->
 
             <!-- ko if: $root.job().mainType() == 'schedules' -->
@@ -292,8 +292,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
   </ul>
 
   <div class="tab-content" id="job-mapreduce-page-logs">
-    <table class="table table-condensed">
-    </table>
+    <pre data-bind="text: logs['default']">
+    </pre>
   </div>
 
   <div class="tab-content" id="job-mapreduce-page-attempts">
@@ -359,10 +359,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 </script>
 
 
-<script type="text/html" id="batch-page">
-  <h2>Workflow</h2>
-  <br/>
-
+<script type="text/html" id="workflow-page">
   ${ _('Id') } <span data-bind="text: id"></span>
   ${ _('Name') } <span data-bind="text: name"></span>
   ${ _('Type') } <span data-bind="text: type"></span>
@@ -371,12 +368,56 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
   ${ _('Progress') } <span data-bind="text: progress"></span>
   ${ _('Duration') } <span data-bind="text: duration"></span>
   ${ _('Submitted') } <span data-bind="text: submitted"></span>
+  
+  </br></br>
+  <span>
+  Variables</br>
+  Duration 8s</br>
+  </br></br>
+  
+  <div class="progress-job progress active pull-left" style="background-color: #FFF; width: 100%" data-bind="css: {'progress-warning': progress() < 100, 'progress-success': progress() === 100}">
+    <div class="bar" data-bind="style: {'width': progress() + '%'}"></div>
+  </div>
+
+  Stop | Resume
+
+  </br>
+
+  <ul class="nav nav-tabs">
+    <li class="active"><a href="#job-mapreduce-page-logs" data-toggle="tab">${ _('Graph') }</a></li>
+    <li><a href="#job-mapreduce-page-attempts" data-toggle="tab">${ _('Logs') }</a></li>
+    <li><a href="#job-mapreduce-page-tasks" data-toggle="tab">${ _('Tasks') }</a></li>
+    <li><a href="#job-mapreduce-page-metadata" data-toggle="tab">${ _('Metadata') }</a></li>
+  </ul>
+
+  <div class="tab-content" id="job-mapreduce-page-logs">
+    <table class="table table-condensed">
+    </table>
+  </div>
+
+  <div class="tab-content" id="job-mapreduce-page-attempts">
+    <table class="table table-condensed">
+    </table>
+  </div>
+
+  <div class="tab-content" id="job-mapreduce-page-tasks">
+    <table class="table table-condensed">
+    </table>
+  </div>
+  
+  <div class="tab-content" id="job-mapreduce-page-metadata">
+    <table class="table table-condensed">
+    </table>
+  </div>
+  
+  <div class="tab-content" id="job-mapreduce-page-counters">
+    <table class="table table-condensed">
+    </table>
+  </div>
+
 </script>
 
 <script type="text/html" id="schedule-page">
-  <h2>Schedule</h2>
-  <br/>
-
   ${ _('Id') } <span data-bind="text: id"></span>
   ${ _('Name') } <span data-bind="text: name"></span>
   ${ _('Type') } <span data-bind="text: type"></span>
@@ -435,6 +476,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       self.duration = ko.observableDefault(job.duration);
       self.submitted = ko.observableDefault(job.submitted);
 
+      self.logs = ko.mapping.fromJS({'default': ''});
+
       self.coordVM = new RunningCoordinatorModel([]);
 
       self.properties = ko.observableDefault(job.properties, {});
@@ -463,6 +506,20 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           self.loadingJob(false);
         });
       };
+
+      self.fetchLogs = function () {
+        $.post("/jobbrowser/api/job/logs", {
+          appid: ko.mapping.toJSON(self.id),
+          interface: ko.mapping.toJSON(vm.interface)
+        }, function (data) {
+          if (data.status == 0) {
+            self.logs['default'](data.logs.logs['default'])
+          } else {
+            $(document).trigger("error", data.message);
+          }
+        }).always(function () {
+        });
+      };
     };
 
     var Jobs = function (vm, options) {
@@ -545,7 +602,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         switch (h) {
           case 'schedules':
           case 'apps':
-          case 'batches':
+          case 'workflows':
             viewModel.interface(h);
             break;
           default:

+ 1 - 0
apps/jobbrowser/src/jobbrowser/urls.py

@@ -56,5 +56,6 @@ urlpatterns += patterns('jobbrowser.views',
 
 urlpatterns += patterns('jobbrowser.api2',
   url(r'api/jobs', 'jobs', name='jobs'),
+  url(r'api/job/logs', 'logs', name='logs'),
   url(r'api/job', 'job', name='job'),
 )