Эх сурвалжийг харах

HUE-8674 [jb] Add timeline to profile.

jdesjean 6 жил өмнө
parent
commit
f3d4ac74cc

+ 1 - 1
apps/impala/src/impala/api.py

@@ -167,7 +167,7 @@ def alanize_metrics(request):
     profile = analyzer.analyze(analyzer.parse_data(query_profile))
     ANALYZER.pre_process(profile)
     metrics = analyzer.metrics(profile)
-    response['data'] = { 'metrics': metrics }
+    response['data'] = metrics
     response['status'] = 0
   return JsonResponse(response)
 

Файлын зөрүү хэтэрхий том тул дарагдсан байна
+ 0 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css


+ 90 - 21
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -13,6 +13,7 @@ function impalaDagre(id) {
   var svg = d3.select("#"+id + " svg");
   var inner = svg.select("g");
   var states_by_name = { };
+  var colors = HueColors.cuiD3Scale();
   var _impalaDagree = {
     _metrics: {},
     init: function (initialScale) {
@@ -21,11 +22,13 @@ function impalaDagre(id) {
       .scale(initialScale)
       .event(svg);
     },
-    metrics: function(metrics) {
-      _impalaDagree._metrics = metrics;
+    metrics: function(data) {
+      _impalaDagree._metrics = data;
+      renderGraph()
     },
     update: function(plan) {
-      renderGraph(plan);
+      _impalaDagree._plan = plan;
+      renderGraph();
       _impalaDagree._width = $(svg[0]).width();
     },
     height: function(value) {
@@ -123,7 +126,7 @@ function impalaDagre(id) {
                   "is_broadcast": node["is_broadcast"],
                   "max_time_val": node["max_time_val"]});
     if (parent) {
-      var label_val = "" + node["output_card"].toLocaleString();
+      var label_val = "" + ko.bindingHandlers.simplesize.humanSize(parseInt(node["output_card"], 10));
       edges.push({ start: node["label"], end: parent,
                    style: { label: label_val }});
     }
@@ -132,7 +135,7 @@ function impalaDagre(id) {
     if (node["data_stream_target"]) {
       edges.push({ "start": node["label"],
                    "end": node["data_stream_target"],
-                   "style": { label: "" + node["output_card"].toLocaleString(),
+                   "style": { label: ko.bindingHandlers.simplesize.humanSize(parseInt(node["output_card"], 10)),
                               style: "stroke-dasharray: 5, 5;"}});
     }
     max_node_time = Math.max(node["max_time_val"], max_node_time)
@@ -200,36 +203,102 @@ function impalaDagre(id) {
     return html;
   }
 
+  function getTimelineData(key) {
+    if (!_impalaDagree._metrics) {
+      return [];
+    }
+    var id = parseInt(key.split(':')[0], 10);
+    if (!_impalaDagree._metrics[id]) {
+      return [];
+    }
+    var times = _impalaDagree._metrics[id];
+    var timesKeys = Object.keys(times);
+    var timesKey;
+    for (var i = 0; i < timesKeys.length; i++) {
+      if (times[timesKeys[i]]['timeline'] && times[timesKeys[i]]['timeline']['Node Lifecycle Event Timeline']) {
+        timesKey = timesKeys[i];
+        break;
+      }
+    }
+    if (!timesKey) {
+      return [];
+    }
+    var time = times[timesKey]['timeline']['Node Lifecycle Event Timeline'];
+    return time.map(function (time, index, array) {
+      var startTime = index > 0 && array[index - 1].value || 0;
+      return { starting_time: startTime, ending_time : time.value, duration: time.value - startTime, color: colors[index % colors.length], name: time.name, unit: time.unit };
+    });
+  }
+
+  function renderTimeline(key) {
+    var datum = getTimelineData(key);
+    var end = _impalaDagree._metrics && _impalaDagree._metrics['max'] || 10;
+    var divider = end > 33554428 ? 1000000 : 1; // values are in NS, scaling to MS as max pixel value is 33554428px ~9h in MS
+    var html = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ' + (end / divider) + ' 10" class="timeline" preserveAspectRatio="none">';
+    html += datum.map(function(time, index) {
+      return '<rect x="' + (time.starting_time / divider) + '" width="' + (time.duration / divider)  + '" height="10" style="fill:' + time.color  +'"></rect>';
+    }).join('');
+    html += '</svg>';
+    return html;
+  }
+
+  function renderTimelineLegend(key) {
+    var datum = getTimelineData(key);
+    var html = '';
+    if (datum) {
+      html += datum.map(function(time, index) {
+        return '<li><div class="legend-icon" style="background-color:' + time.color +' "></div><div class="metric-name">' + time.name + '</div> <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(time.duration, time.unit) + '</div></li>';
+      }).join('');
+    }
+    html += '';
+    return html;
+  }
+
   function showDetail(id) {
     var data;
-    if (_impalaDagree._metrics[id] && _impalaDagree._metrics[id]['averaged']) {
-      data = _impalaDagree._metrics[id]['averaged'];
-    } else if (_impalaDagree._metrics[id]) {
-      data = _impalaDagree._metrics[id][Object.keys(_impalaDagree._metrics[id])[0]];
+    if (_impalaDagree._metrics[id] && _impalaDagree._metrics[id]['averaged']['metrics']) {
+      data = _impalaDagree._metrics[id]['averaged']['metrics'];
+    } else if (_impalaDagree._metrics[id]['metrics']) {
+      data = _impalaDagree._metrics[id]['metrics'][Object.keys(_impalaDagree._metrics[id]['metrics'])[0]];
     }
     d3.select('.query-plan').classed('open', true);
-    var title = d3.select('.query-plan .details')
-    .selectAll('.metric-title').data([0]);
-    title.enter().append('div').classed('metric-title', true);
+    var details = d3.select('.query-plan .details');
     var key = getKey(id);
-    title.html(getIcon(states_by_name[key].icon) + '<span>' + states_by_name[key].label+ '</span>');
+    details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h3>' + states_by_name[key].label+ '</h3></div>')
+    var detailsContent = details.append('div').classed('details-content', true);
+    var timelineSection = detailsContent.append('div').classed('details-section', true);
+
+    var timelineTitle = timelineSection.append('header');
+    timelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
+    timelineTitle.append('h4').text(window.HUE_I18n.profile.timeline);
+
+    timelineSection.node().appendChild($.parseXML(renderTimeline(key, '')).children[0]);
+
+    timelineSection.append('ol').classed('', true).html(renderTimelineLegend(key));
+
+    detailsContent.append('div').classed('divider', true);
+
+    var metricsSection = detailsContent.append('div').classed('details-section', true);
+
+    var metricsTitle = metricsSection.append('header');
+    metricsTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-bar-chart');
+    metricsTitle.append('h4').text(window.HUE_I18n.profile.metrics);
 
-    var metricTitle = d3.select('.query-plan .details')
-    .selectAll('.metrics').data([0]);
-    metricTitle.enter().append('div').classed('metrics', true);
+    var metricsContent = metricsSection.append('ul').classed('metrics', true);
 
-    var metrics = d3.select('.query-plan .details .metrics').selectAll('div')
+    var metrics = metricsContent.selectAll('li')
     .data(Object.keys(data).sort().map(function (key) { return data[key]; }));
     metrics.exit().remove();
-    metrics.enter().append('div');
-    metrics.html(function (datum) { return '<div class="metric-name">' + datum.name + '</div> : <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(datum.value, datum.unit) + '</div>'; });
+    metrics.enter().append('li');
+    metrics.html(function (datum) { return '<div class="metric-name">' + datum.name + '</div> <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(datum.value, datum.unit) + '</div>'; });
   }
 
   function hideDetail(id) {
     d3.select('.query-plan').classed('open', false);
   }
 
-  function renderGraph(plan) {
+  function renderGraph() {
+    var plan = _impalaDagree._plan;
     if (!plan || !plan.plan_nodes || !plan.plan_nodes.length) return;
     var states = [];
     var edges = [];
@@ -251,8 +320,8 @@ function impalaDagre(id) {
       html += "<span class='name'>" + state.label + "</span><br/>";
       html += "<span class='metric'>" + state.max_time + "</span>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
-      html += "<span class='metric'>" + state.max_time + "</span>"
       html += "<span class='id'>" + state.name + "</span>";
+      html += renderTimeline(state.name);
       html += "</div>";
 
       var style = state.style;

+ 49 - 8
apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less

@@ -111,7 +111,9 @@
       overflow: hidden;
       text-overflow: ellipsis;
       width: calc(~"100% - 32px");
+      max-width: 185px;
       display: inline-block;
+      height: 14px;
     }
     foreignObject {
       .fa {
@@ -198,6 +200,12 @@
     foreignObject > div {
       position: relative;
     }
+    .timeline {
+      width: 175px;
+      height: 10px;
+      border: 1px solid @cui-gray-300;
+      border-radius: 4px;
+    }
     .details {
       background-color: white;
       box-shadow: 0px 0px 10px 0px;
@@ -208,12 +216,38 @@
       right: 0px;
       width: 200px;
       height: 100%;
+      flex-direction: column;
+      .divider {
+        background-color: @cui-gray-300;
+        width: calc(~"100% - 10px");
+        margin-left: 5px;
+        padding: 0px;
+        height: 1px;
+      }
+      h3 {
+        display: inline-block;
+        margin: 0px;
+      }
+      h4 {
+        margin: 0px;
+        display: inline-block;
+      }
+      ol {
+        list-style-type: none;
+        margin: 0px;
+      }
+      ul {
+        margin: 0px;
+      }
+      .details-content {
+        overflow: scroll;
+      }
+      .details-section {
+        padding: 4px;
+      }
       .metric-title {
         background-color: @hue-primary-color-light;
-        font-weight: bold;
-        font-size: 16px;
-        color: @cui-gray-800;
-        line-height: 40px;
+        padding: 4px;
         .fa {
           padding-top: 0px;
           line-height: 40px;
@@ -226,11 +260,11 @@
       }
       .metrics {
         overflow-y: scroll;
-        height: calc(~"100% - 40px");
+        max-height: calc(~"100% - 40px");
       }
       .metric-name {
         color: @cui-gray-800;
-        width: 120px;
+        width: 107px;
         overflow: hidden;
         text-align: right;
         display: inline-block;
@@ -242,15 +276,22 @@
       .metric-value {
         color: @cui-gray-800;
         overflow: hidden;
-        width: 50px;
+        width: 60px;
         vertical-align: middle;
         display: inline-block;
         white-space: nowrap;
       }
+      .legend-icon {
+        width: 8px;
+        height: 8px;
+        border-radius: 5px;
+        border: 1px solid @cui-gray-300;
+        display: inline-block;
+      }
     }
   }
   .query-plan.open .details {
-    display: inherit;
+    display: flex;
   }
   .query-plan.open .buttons {
     right: 208px;

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -1534,7 +1534,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       <div class="tab-content">
         <div class="tab-pane" id="queries-page-plan${ SUFFIX }" data-profile="plan">
           <div data-bind="visible:properties.plan && properties.plan().plan_json && properties.plan().plan_json.plan_nodes.length">
-            <div class="query-plan" id="queries-page-plan-graph${ SUFFIX }" data-bind="impalaDagre: {value: properties.plan && properties.plan().plan_json, metrics: properties.metrics && properties.metrics().metrics, height:$root.isMini() ? 535 : 600 }">
+            <div class="query-plan" id="queries-page-plan-graph${ SUFFIX }" data-bind="impalaDagre: {value: properties.plan && properties.plan().plan_json, metrics: properties.metrics && properties.metrics(), height:$root.isMini() ? 535 : 600 }">
               <svg style="width:100%;height:100%;" id="queries-page-plan-svg${ SUFFIX }">
                 <defs>
                   <filter id="dropshadow" height="130%">

+ 54 - 10
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -7539,12 +7539,20 @@
       },
       human: function (value, unit) {
         var fn;
-        if (unit == 3) {
+        if (unit === 1) {
+          fn = function(value) { return ko.bindingHandlers.simplesize.humanSize(value) + '/s'; }
+        } else if (unit === 3) {
           fn = ko.bindingHandlers.bytesize.humanSize
-        } else if (unit == 5) {
+        } else if (unit === 4) {
+          fn = function(value) { return ko.bindingHandlers.bytesize.humanSize(value) + '/s'; }
+        } else if (unit === 5) {
           fn = ko.bindingHandlers.duration.humanTime
+        } else if (unit === 8) {
+          fn = function (value) { return ko.bindingHandlers.duration.humanTime(value * 1000000); }
+        } else if (unit === 9) {
+          fn = function (value) { return ko.bindingHandlers.duration.humanTime(value * 1000000000); }
         } else {
-          fn = function(value){ return value; }
+          fn = ko.bindingHandlers.simplesize.humanSize
         }
         return fn(value);
       }
@@ -7570,17 +7578,17 @@
         if (value < Math.pow(10, 3)) {
           return value + " ns";
         } else if (value < Math.pow(10, 6)) {
-          value = (value * 1.0) / Math.pow(10, 6);
-          return sprintf("%.4f ms", value);
+          value = (value * 1.0) / Math.pow(10, 3);
+          return sprintf("%.2f us", value);
         } else if (value < Math.pow(10, 9)) {
-          value = (value * 1.0) / Math.pow(10, 9);
-          return sprintf("%.4f s", value);
+          value = (value * 1.0) / Math.pow(10, 6);
+          return sprintf("%.2f ms", value);
         } else {
           // get the ms value
-          var SECOND = 1000;
+          var SECOND = 1;
           var MINUTE = SECOND * 60;
           var HOUR = MINUTE * 60;
-          var value = value * 1 / Math.pow(10, 6);
+          value = value * 1 / Math.pow(10, 9);
           var buffer = "";
 
           if (value > (HOUR)) {
@@ -7594,7 +7602,7 @@
           }
 
           if (value > SECOND) {
-            buffer += sprintf("%.3f s", value * 1.0 / SECOND);
+            buffer += sprintf("%.2f s", value * 1.0 / SECOND);
           }
           return buffer;
         }
@@ -7638,4 +7646,40 @@
     };
   })();
 
+  ko.bindingHandlers.simplesize = (function() {
+    var that;
+    return that = {
+      units: ["", "K", "M", "G", "T", "P"],
+      init: function (element, valueAccessor) {
+        that.format(element, valueAccessor);
+      },
+      update: function (element, valueAccessor) {
+        that.format(element, valueAccessor);
+      },
+      format: function (element, valueAccessor) {
+        var value = valueAccessor();
+        var formatted = that.humanSize(ko.unwrap(value));
+        $(element).text(formatted);
+      },
+      getBaseLog: function(x, y) {
+        return Math.log(x) / Math.log(y);
+      },
+      humanSize: function(bytes) {
+        var isNumber = !isNaN(parseFloat(bytes)) && isFinite(bytes);
+        if (!isNumber) {
+          return '';
+        }
+
+        // Special case small numbers (including 0), because they're exact.
+        if (bytes < 1000) {
+          return sprintf("%d", bytes);
+        }
+
+        var index = Math.floor(that.getBaseLog(bytes, 1000));
+        index = Math.min(that.units.length - 1, index);
+        return sprintf("%.1f %s", bytes / Math.pow(1000, index), that.units[index])
+      }
+    };
+  })();
+
 })();

+ 4 - 0
desktop/core/src/desktop/templates/global_js_constants.mako

@@ -175,6 +175,10 @@
       missingLongitude: "${ _('Missing longitude configuration.') }",
       missingLabel: "${ _('Missing label configuration.') }",
       missingRegion: "${ _('Missing region configuration.') }",
+    },
+    profile: {
+      timeline: "${ _('Timeline') }",
+      metrics: "${ _('Metrics') }",
     }
   };
 

+ 6 - 0
desktop/core/src/desktop/templates/hue_icons.mako

@@ -353,6 +353,12 @@
     <symbol id="hi-crop-free" viewBox="0 0 24 24">
       <path d="M3 5v4h2V5h4V3H5c-1.1 0-2 .9-2 2zm2 10H3v4c0 1.1.9 2 2 2h4v-2H5v-4zm14 4h-4v2h4c1.1 0 2-.9 2-2v-4h-2v4zm0-16h-4v2h4v4h2V5c0-1.1-.9-2-2-2z"/>
     </symbol>
+    <symbol id="hi-bar-chart" viewBox="0 0 24 24">
+      <path d="M5 9.2h3V19H5zM10.6 5h2.8v14h-2.8zm5.6 8H19v6h-2.8z"/><path fill="none" d="M0 0h24v24H0z"/>
+    </symbol>
+    <symbol id="hi-access-time" viewBox="0 0 24 24">
+      <path d="M11.99 2C6.47 2 2 6.48 2 12s4.47 10 9.99 10C17.52 22 22 17.52 22 12S17.52 2 11.99 2zM12 20c-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8-3.58 8-8 8z"/><path d="M0 0h24v24H0z" fill="none"/><path d="M12.5 7H11v6l5.25 3.15.75-1.23-4.5-2.67z"/>
+    </symbol>
   </svg>
 
   <script type="text/html" id="app-switcher-icon-template">

+ 21 - 4
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -28,6 +28,7 @@ from thrift.transport import TTransport
 from libanalyze import dot
 from libanalyze import gjson as jj
 from libanalyze import models
+from libanalyze.rules import to_double
 
 
 class Node(object):
@@ -234,9 +235,20 @@ class Node(object):
     ctr = {}
     if self.val.counters:
         for c in self.val.counters:
-            ctr[c.name] = { 'name': c.name, 'value': c.value, 'unit': c.unit }
+            ctr[c.name] = { 'name': c.name, 'value': to_double(c.value) if c.unit == 6 else c.value, 'unit': c.unit }
     return ctr
 
+  def event_list(self):
+    event_list = {}
+    if self.val.event_sequences:
+      for s in self.val.event_sequences:
+        sequence_name = s.name
+        event_list[sequence_name] = []
+        for i in range(len(s.labels)):
+          event_name = s.labels[i]
+          event_list[sequence_name].append({'name': event_name, 'value': s.timestamps[i], 'unit': 5})
+    return event_list
+
   def repr(self, indent):
     buffer = indent + self.val.name + "\n"
     if self.val.info_strings:
@@ -284,7 +296,7 @@ def metrics(profile):
   execution_profile = profile.find_by_name('Execution Profile')
   if not execution_profile:
     return {}
-  counter_map = {}
+  counter_map = {'max': 0}
   def get_metric(node, counter_map=counter_map):
     if not node.is_plan_node():
       return
@@ -292,11 +304,16 @@ def metrics(profile):
     if counter_map.get(nid) is None:
       counter_map[nid] = {}
     host = node.augmented_host()
+    event_list = node.event_list();
+    if event_list and event_list.get('Node Lifecycle Event Timeline'):
+      last_value = event_list['Node Lifecycle Event Timeline'][len(event_list['Node Lifecycle Event Timeline']) - 1]['value']
+      counter_map['max'] = max(last_value, counter_map['max'])
     if host:
-      counter_map[nid][host] = node.metric_map()
+      counter_map[nid][host] = {'metrics': node.metric_map(), 'timeline': event_list}
     else:
-      counter_map[nid] = node.metric_map()
+      counter_map[nid] = {'metrics': node.metric_map(), 'timeline': event_list}
   execution_profile.foreach_lambda(get_metric)
+  counter_map['ImpalaServer'] = profile.find_by_name('ImpalaServer').metric_map()
   return counter_map
 
 def heatmap_by_host(profile, counter_name):

+ 3 - 2
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -561,7 +561,6 @@ class TopDownAnalysis:
               for i in range(len(s.labels)):
                 event_name = s.labels[i]
                 event_duration = s.timestamps[i] - duration
-                event_value = s.timestamps[i]
 
                 if sequence.get(event_name):
                   summary.val.counters.append(models.TCounter(name=sequence.get(event_name), value=event_duration, unit=5))
@@ -619,7 +618,9 @@ class TopDownAnalysis:
             # Make sure to substract the wait time for the exchange node
             if is_plan_node and re.search(r'EXCHANGE_NODE', node.val.name) is not None:
                 async_time = counter_map.get('AsyncTotalTime', models.TCounter(value=0)).value
-                local_time = counter_map['TotalTime'].value - counter_map['InactiveTotalTime'].value - async_time
+                dequeue = node.find_by_name('Dequeue')
+                data_wait_time = dequeue.counter_map().get('DataWaitTime', models.TCounter(value=0)).value if dequeue else 0
+                local_time = counter_map['TotalTime'].value - counter_map['InactiveTotalTime'].value - async_time - data_wait_time
 
             # For Hash Join, if the "LocalTime" metrics
             if is_plan_node and re.search(r'HASH_JOIN_NODE', node.val.name) is not None:

Энэ ялгаанд хэт олон файл өөрчлөгдсөн тул зарим файлыг харуулаагүй болно