Explorar el Código

HUE-1302 [oozie] Optimize dashboards API calls for a single coordinator

Display a maximum of 20 pages of logs for Workflow, Coordinator and Bundles.
Display a maxium of 250 coordinator actions.
Possibility to display all the coordinator actions by ckicking on a button.
Refresh times increased to 20s for Coordinators and Bundles
Romain Rigaux hace 12 años
padre
commit
ab09a82

+ 4 - 4
apps/oozie/src/oozie/templates/dashboard/list_oozie_bundle.mako

@@ -137,7 +137,7 @@ ${ layout.menubar(section='bundles', dashboard=True) }
       <li><a href="#definition" data-toggle="tab">${ _('Definition') }</a></li>
     </ul>
 
-    <div class="tab-content" style="padding-bottom:200px">
+    <div class="tab-content" style="min-height:200px">
       <div class="tab-pane active" id="calendar">
         <table class="table table-striped table-condensed">
           <thead>
@@ -466,8 +466,8 @@ ${ layout.menubar(section='bundles', dashboard=True) }
         $("#progress .bar").text(data.progress + "%").css("width", data.progress + "%").attr("class", "bar " + getStatusClass(data.status, "bar-"));
 
         var _logsEl = $("#log pre");
-        var newLines = data.log.split("\n").slice(_logsEl.text().split("\n").length);
-        _logsEl.text(_logsEl.text() + newLines.join("\n"));
+        _logsEl.text(data.log);
+
         if (logsAtEnd) {
           _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height());
         }
@@ -475,7 +475,7 @@ ${ layout.menubar(section='bundles', dashboard=True) }
         if (data.status != "RUNNING" && data.status != "PREP"){
           return;
         }
-        window.setTimeout(refreshView, 1000);
+        window.setTimeout(refreshView, 20000);
       });
     }
 

+ 1 - 1
apps/oozie/src/oozie/templates/dashboard/list_oozie_bundles.mako

@@ -399,7 +399,7 @@ ${layout.menubar(section='bundles', dashboard=True)}
         }
         numRunning = data.length;
 
-        window.setTimeout(refreshRunning, 1000);
+        window.setTimeout(refreshRunning, 20000);
       });
     }
 

+ 17 - 5
apps/oozie/src/oozie/templates/dashboard/list_oozie_coordinator.mako

@@ -144,7 +144,7 @@ ${ layout.menubar(section='coordinators', dashboard=True) }
       % endif
     </ul>
 
-    <div class="tab-content" style="padding-bottom:200px">
+    <div class="tab-content" style="min-height:200px">
       <div class="tab-pane active" id="calendar">
         <table class="table table-striped table-condensed">
           <thead>
@@ -156,6 +156,15 @@ ${ layout.menubar(section='coordinators', dashboard=True) }
           <tbody data-bind="template: {name: 'calendarTemplate', foreach: actions}">
           </tbody>
           <tfoot>
+            <tr>
+              <td data-bind="visible: !isLoading() && paginate()" colspan="10">
+                </br>
+                <div class="alert">
+                  ${ _('There are older actions to be shown:') }
+                  <a class="btn" href="${ oozie_coordinator.get_absolute_url() }?show_all_actions=true">${ _('Expand') }</a>
+                </div>
+              </td>
+            </tr>
             <tr data-bind="visible: isLoading()">
               <td colspan="2" class="left">
                 <img src="/static/art/spinner.gif" />
@@ -401,6 +410,9 @@ ${ layout.menubar(section='coordinators', dashboard=True) }
     self.actions = ko.observableArray(ko.utils.arrayMap(actions), function (action) {
       return new Action(action);
     });
+    self.paginate = ko.computed(function(){
+      return self.actions().length >= ${ MAX_COORD_ACTIONS } && ${ "false" if show_all_actions else 'true' | n,unicode };
+    });
   };
 
   var viewModel = new RunningCoordinatorActionsModel([]);
@@ -526,7 +538,7 @@ ${ layout.menubar(section='coordinators', dashboard=True) }
     var logsAtEnd = true;
 
     function refreshView() {
-      $.getJSON("${ oozie_coordinator.get_absolute_url(oozie_bundle) }" + "?format=json", function (data) {
+      $.getJSON("${ oozie_coordinator.get_absolute_url(oozie_bundle) }" + "?format=json" + "${ "&show_all_actions=true" if show_all_actions else '' | n,unicode }", function (data) {
         viewModel.isLoading(false);
         if (data.actions){
           viewModel.actions(ko.utils.arrayMap(data.actions, function (action) {
@@ -560,15 +572,15 @@ ${ layout.menubar(section='coordinators', dashboard=True) }
         $("#progress .bar").text(data.progress + "%").css("width", data.progress + "%").attr("class", "bar " + getStatusClass(data.status, "bar-"));
 
         var _logsEl = $("#log pre");
-        var newLines = data.log.split("\n").slice(_logsEl.text().split("\n").length);
-        _logsEl.text(_logsEl.text() + newLines.join("\n"));
+        _logsEl.text(data.log);
+
         if (logsAtEnd) {
           _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height());
         }
         if (data.status != "RUNNING" && data.status != "PREP"){
           return;
         }
-        window.setTimeout(refreshView, 1000);
+        window.setTimeout(refreshView, 20000);
       });
     }
 

+ 1 - 1
apps/oozie/src/oozie/templates/dashboard/list_oozie_coordinators.mako

@@ -412,7 +412,7 @@ ${layout.menubar(section='coordinators', dashboard=True)}
         }
         numRunning = data.length;
 
-        window.setTimeout(refreshRunning, 1000);
+        window.setTimeout(refreshRunning, 20000);
       });
     }
 

+ 3 - 2
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflow.mako

@@ -619,8 +619,9 @@ ${ layout.menubar(section='workflows', dashboard=True) }
         $("#graph").html(data.graph);
 
         var _logsEl = $("#log pre");
-        var newLines = data.log.split("\n").slice(_logsEl.text().split("\n").length);
-        _logsEl.text(_logsEl.text() + newLines.join("\n"));
+
+        _logsEl.text(data.log);
+
         if (logsAtEnd) {
           _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height());
         }

+ 11 - 2
apps/oozie/src/oozie/views/dashboard.py

@@ -46,7 +46,7 @@ from oozie.settings import DJANGO_APPS
 
 
 LOG = logging.getLogger(__name__)
-
+MAX_COORD_ACTIONS = 250
 
 """
 Permissions:
@@ -245,7 +245,13 @@ def list_oozie_coordinator(request, job_id, bundle_job_id=None):
   if bundle_job_id is not None:
     oozie_bundle = check_job_access_permission(request, bundle_job_id)
 
+  show_all_actions =request.GET.get('show_all_actions') == 'true'
+
   if request.GET.get('format') == 'json':
+    actions = massaged_coordinator_actions_for_json(oozie_coordinator, oozie_bundle)
+    if not show_all_actions:
+      actions = actions[:MAX_COORD_ACTIONS]
+
     return_obj = {
       'id': oozie_coordinator.id,
       'status':  oozie_coordinator.status,
@@ -253,7 +259,8 @@ def list_oozie_coordinator(request, job_id, bundle_job_id=None):
       'nextTime': format_time(oozie_coordinator.nextMaterializedTime),
       'endTime': format_time(oozie_coordinator.endTime),
       'log': oozie_coordinator.log,
-      'actions': massaged_coordinator_actions_for_json(oozie_coordinator, oozie_bundle)
+      'actions': actions,
+      'show_all_actions': show_all_actions
     }
     return HttpResponse(encode_json_for_js(return_obj), mimetype="application/json")
 
@@ -271,6 +278,8 @@ def list_oozie_coordinator(request, job_id, bundle_job_id=None):
     'coordinator': coordinator,
     'oozie_bundle': oozie_bundle,
     'has_job_edition_permission': has_job_edition_permission,
+    'show_all_actions': show_all_actions,
+    'MAX_COORD_ACTIONS': MAX_COORD_ACTIONS
   })
 
 

+ 8 - 4
desktop/libs/liboozie/src/liboozie/types.py

@@ -270,9 +270,13 @@ class BundleAction(Action):
 
 
 class Job(object):
-  RUNNING_STATUSES = set(['PREP', 'RUNNING', 'SUSPENDED', # Workflow
-                          'RUNNING', 'PREPSUSPENDED', 'SUSPENDED', 'PREPPAUSED', 'PAUSED' # Coordinator
-                          ])
+  RUNNING_STATUSES = set([
+     'PREP', 'RUNNING', 'SUSPENDED', # Workflow
+     'RUNNING', 'PREPSUSPENDED', 'SUSPENDED', 'PREPPAUSED', 'PAUSED' # Coordinator
+    ]
+  )
+  MAX_LOG_SIZE = 3500 * 20 # 20 pages
+
   """
   Accessing log and definition will trigger Oozie API calls.
   """
@@ -311,7 +315,7 @@ class Job(object):
     """Get the log lazily, trigger Oozie API call at the first access."""
     if self._log is None:
       self._log = self._api.get_job_log(self.id)
-    return self._log
+    return self._log[-Job.MAX_LOG_SIZE:]
   log = property(_get_log)
 
   def _get_definition(self):