Browse Source

HUE-2824 [oozie] Improve dashboard response time for larger job count

1. Avoiding multiple timer objects being created - decreases unnecessary requests to server
2. Earlier, there were two steps when updating Running table
   a) checking for zombie nodes [O(n^2)]
   b) Update the existing jobs [O(n^2)]

   Now,
   a) We just get the previously selected job-ids (more often than not is very small)
   b) Clear previous data
   c) When inserting the new job, we check if it's in this list (to restore selected jobs)
   d) Also, using a map to restore progress information to make it seamless.

So, we decrease the update time to approx. O(n). Seems like there is significant improvement in response time.
krish 10 years ago
parent
commit
60a3a6a277
1 changed files with 50 additions and 44 deletions
  1. 50 44
      apps/oozie/src/oozie/templates/dashboard/list_oozie_workflows.mako

+ 50 - 44
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflows.mako

@@ -213,7 +213,7 @@ ${ layout.menubar(section='workflows', dashboard=True) }
     }
     }
   }
   }
 
 
-  var refreshRunning;
+  var refreshRunning, runningTimeout, progressTimeout, jobProgressMap = {};
   var runningTableOffset = 1, completedTableOffset = 1;
   var runningTableOffset = 1, completedTableOffset = 1;
   var totalRunningJobs = 0, totalCompletedJobs = 0;
   var totalRunningJobs = 0, totalCompletedJobs = 0;
   var PAGE_SIZE = 50;
   var PAGE_SIZE = 50;
@@ -318,6 +318,7 @@ ${ layout.menubar(section='workflows', dashboard=True) }
         if ($(this).data("table") == "running") {
         if ($(this).data("table") == "running") {
           runningTableOffset += _additionalOffset;
           runningTableOffset += _additionalOffset;
           refreshRunning();
           refreshRunning();
+          refreshProgress();
         }
         }
         else {
         else {
           completedTableOffset += _additionalOffset;
           completedTableOffset += _additionalOffset;
@@ -331,6 +332,7 @@ ${ layout.menubar(section='workflows', dashboard=True) }
       $(this).toggleClass("active");
       $(this).toggleClass("active");
       refreshRunning();
       refreshRunning();
       refreshCompleted();
       refreshCompleted();
+      refreshProgress();
     });
     });
 
 
     $("a.btn-submitted").click(function () {
     $("a.btn-submitted").click(function () {
@@ -345,6 +347,7 @@ ${ layout.menubar(section='workflows', dashboard=True) }
       $(this).toggleClass("active");
       $(this).toggleClass("active");
       refreshRunning();
       refreshRunning();
       refreshCompleted();
       refreshCompleted();
+      refreshProgress();
     });
     });
 
 
     var hash = window.location.hash.replace(/(<([^>]+)>)/ig, "");
     var hash = window.location.hash.replace(/(<([^>]+)>)/ig, "");
@@ -355,6 +358,9 @@ ${ layout.menubar(section='workflows', dashboard=True) }
     function refreshPagination() {
     function refreshPagination() {
       runningTableOffset = 1;
       runningTableOffset = 1;
       completedTableOffset = 1;
       completedTableOffset = 1;
+
+      // Clear select-all
+      $(".hueCheckbox").removeClass("fa-check");
     }
     }
 
 
     function drawTable() {
     function drawTable() {
@@ -387,7 +393,6 @@ ${ layout.menubar(section='workflows', dashboard=True) }
       if (btnStatuses.length > 0) {
       if (btnStatuses.length > 0) {
         selectedStatuses = $.makeArray($(selectedStatuses).filter(btnStatuses));
         selectedStatuses = $.makeArray($(selectedStatuses).filter(btnStatuses));
       }
       }
-
       return selectedStatuses.length > 0 ? ('&status=' + selectedStatuses.join('&status=')) : '';
       return selectedStatuses.length > 0 ? ('&status=' + selectedStatuses.join('&status=')) : '';
     }
     }
 
 
@@ -433,6 +438,7 @@ ${ layout.menubar(section='workflows', dashboard=True) }
     var numRunning = 0;
     var numRunning = 0;
 
 
     refreshRunning = function () {
     refreshRunning = function () {
+      window.clearTimeout(runningTimeout);
       $.getJSON(window.location.pathname + "?format=json&offset=" + runningTableOffset + getStatuses('running') + getDaysFilter(), function (data) {
       $.getJSON(window.location.pathname + "?format=json&offset=" + runningTableOffset + getStatuses('running') + getDaysFilter(), function (data) {
         if (data.jobs.length > 0) {
         if (data.jobs.length > 0) {
           totalRunningJobs = data.total_jobs;
           totalRunningJobs = data.total_jobs;
@@ -440,52 +446,47 @@ ${ layout.menubar(section='workflows', dashboard=True) }
 
 
           var nNodes = runningTable.fnGetNodes();
           var nNodes = runningTable.fnGetNodes();
 
 
-          // check for zombie nodes
-          $(nNodes).each(function (iNode, node) {
-            var nodeFound = false;
-            $(data.jobs).each(function (iWf, currentItem) {
-              if ($(node).children("td").eq(7).text() == currentItem.id) {
-                nodeFound = true;
-              }
-            });
-            if (!nodeFound) {
-              runningTable.fnDeleteRow(node);
-              runningTable.fnDraw();
-            }
+          // Find previously selected jobs
+          var _ids = [];
+          $(".hueCheckbox.fa-check:not(.select-all)").each(function(){
+            _ids.push($(this).parents("tr").find("a[data-row-selector='true']").text());
           });
           });
+          runningTable.fnClearTable();
 
 
           $(data.jobs).each(function (iWf, item) {
           $(data.jobs).each(function (iWf, item) {
             var wf = new Workflow(item);
             var wf = new Workflow(item);
-            var foundRow = null;
-            $(nNodes).each(function (iNode, node) {
-              if ($(node).children("td").eq(7).text() == wf.id) {
-                foundRow = node;
+
+            // Restore previously selected jobs
+            var foundRow = _ids.indexOf(wf.id) != -1;
+
+            var checkboxSelected = "";
+            if (foundRow) {
+              checkboxSelected = "fa-check";
+            }
+            var progressColumn = '<div class="progress"><div class="bar bar-warning" style="width: 1%"></div></div>';
+            if (wf.id in jobProgressMap) {
+              progressColumn = '<div class="progress"><div class="' + jobProgressMap[wf.id]["progressClass"] + '" style="width:' + jobProgressMap[wf.id]["progress"] + '%">' + jobProgressMap[wf.id]["progress"] + '%</div></div>';
+            }
+
+            if (['RUNNING', 'PREP', 'WAITING', 'SUSPENDED', 'PREPSUSPENDED', 'PREPPAUSED', 'PAUSED', 'STARTED', 'FINISHING'].indexOf(wf.status) > -1) {
+              try {
+                runningTable.fnAddData([
+                  wf.canEdit ? '<div class="hueCheckbox fa ' + checkboxSelected + '" data-row-selector-exclude="true"></div>':'',
+                  '<span data-sort-value="'+ wf.createdInMillis +'" data-type="date">' + emptyStringIfNull(wf.created) + '</span>',
+                  '<span class="' + wf.statusClass + '" data-type="status">' + wf.status + '</span>',
+                  wf.appName,
+                  progressColumn,
+                  wf.user,
+                  '<span data-sort-value="'+ wf.lastModTimeInMillis +'">' + emptyStringIfNull(wf.lastModTimeFormatted) + '</span>',
+                  '<a href="' + wf.absoluteUrl + '" data-row-selector="true">' + wf.id + '</a>',
+                  wf.parentUrl == '' ? '' : '<div style="text-align:center"><a href="' + wf.parentUrl + '" style="text-align:center"><img src="' + getParentImage(wf.parentUrl) + '" class="app-icon"/></a></div>',
+                  wf.submittedManually
+                ]);
               }
               }
-            });
-            if (foundRow == null) {
-              if (['RUNNING', 'PREP', 'WAITING', 'SUSPENDED', 'PREPSUSPENDED', 'PREPPAUSED', 'PAUSED', 'STARTED', 'FINISHING'].indexOf(wf.status) > -1) {
-                try {
-                  runningTable.fnAddData([
-                    wf.canEdit ? '<div class="hueCheckbox fa" data-row-selector-exclude="true"></div>':'',
-                    '<span data-sort-value="'+ wf.createdInMillis +'" data-type="date">' + emptyStringIfNull(wf.created) + '</span>',
-                    '<span class="' + wf.statusClass + '" data-type="status">' + wf.status + '</span>',
-                    wf.appName,
-                    '<div class="progress"><div class="bar bar-warning" style="width: 1%"></div></div>',
-                    wf.user,
-                    '<span data-sort-value="'+ wf.lastModTimeInMillis +'">' + emptyStringIfNull(wf.lastModTimeFormatted) + '</span>',
-                    '<a href="' + wf.absoluteUrl + '" data-row-selector="true">' + wf.id + '</a>',
-                    wf.parentUrl == '' ? '' : '<div style="text-align:center"><a href="' + wf.parentUrl + '" style="text-align:center"><img src="' + getParentImage(wf.parentUrl) + '" class="app-icon"/></a></div>',
-                    wf.submittedManually
-                  ]);
-                }
-                catch (error) {
-                  $(document).trigger("error", error);
-                }
+              catch (error) {
+                $(document).trigger("error", error);
               }
               }
             }
             }
-            else {
-              runningTable.fnUpdate('<span class="' + wf.statusClass + '" data-type="status">' + wf.status + '</span>', foundRow, 2, false);
-            }
           });
           });
         }
         }
         if (data.jobs.length == 0) {
         if (data.jobs.length == 0) {
@@ -496,7 +497,8 @@ ${ layout.menubar(section='workflows', dashboard=True) }
         }
         }
         numRunning = data.jobs.length;
         numRunning = data.jobs.length;
 
 
-        window.setTimeout(refreshRunning, 5000);
+        runningTable.fnDraw();
+        runningTimeout = window.setTimeout(refreshRunning, 5000);
       });
       });
     }
     }
 
 
@@ -570,11 +572,15 @@ ${ layout.menubar(section='workflows', dashboard=True) }
     }
     }
 
 
     function refreshProgress() {
     function refreshProgress() {
-      $.getJSON(window.location.pathname + "?format=json&type=progress" + getStatuses('running') + getDaysFilter(), function (data) {
+      window.clearTimeout(progressTimeout);
+      $.getJSON(window.location.pathname + "?format=json&type=progress&offset=" + runningTableOffset + getStatuses('running') + getDaysFilter(), function (data) {
         var nNodes = runningTable.fnGetNodes();
         var nNodes = runningTable.fnGetNodes();
         $(data.jobs).each(function (iWf, item) {
         $(data.jobs).each(function (iWf, item) {
             var wf = new Workflow(item);
             var wf = new Workflow(item);
             var foundRow = null;
             var foundRow = null;
+
+            // Remember job progress info
+            jobProgressMap[wf.id] = {"progressClass": wf.progressClass, "progress": wf.progress};
             $(nNodes).each(function (iNode, node) {
             $(nNodes).each(function (iNode, node) {
               if ($(node).children("td").eq(7).text() == wf.id) {
               if ($(node).children("td").eq(7).text() == wf.id) {
                 foundRow = node;
                 foundRow = node;
@@ -590,7 +596,7 @@ ${ layout.menubar(section='workflows', dashboard=True) }
               }
               }
             }
             }
           });
           });
-        window.setTimeout(refreshProgress, 20000);
+        progressTimeout = window.setTimeout(refreshProgress, 20000);
       });
       });
     }
     }