Bläddra i källkod

HUE-8704 [jb] Add health risks to profile.

jdesjean 6 år sedan
förälder
incheckning
96fbe70bd4
27 ändrade filer med 204 tillägg och 73 borttagningar
  1. 1 1
      apps/impala/src/impala/api.py
  2. 15 3
      apps/jobbrowser/src/jobbrowser/apis/query_api.py
  3. 0 0
      apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css
  4. 76 26
      apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js
  5. 9 0
      apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less
  6. 7 6
      desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js
  7. 3 1
      desktop/core/src/desktop/templates/global_js_constants.mako
  8. 7 0
      desktop/core/src/desktop/templates/hue_icons.mako
  9. 1 0
      desktop/libs/libanalyze/reasons/agg_performance.json
  10. 1 1
      desktop/libs/libanalyze/reasons/bytes_read_skew.json
  11. 1 0
      desktop/libs/libanalyze/reasons/join_performance.json
  12. 1 1
      desktop/libs/libanalyze/reasons/metadata_missing.json
  13. 1 1
      desktop/libs/libanalyze/reasons/remote_scan_ranges.json
  14. 1 1
      desktop/libs/libanalyze/reasons/rows_read_skew.json
  15. 1 1
      desktop/libs/libanalyze/reasons/scan_performance.json
  16. 2 1
      desktop/libs/libanalyze/reasons/scanner_filter.json
  17. 1 1
      desktop/libs/libanalyze/reasons/scanner_parallelism.json
  18. 1 0
      desktop/libs/libanalyze/reasons/selective_scan.json
  19. 1 1
      desktop/libs/libanalyze/reasons/skew.json
  20. 1 1
      desktop/libs/libanalyze/reasons/slow_table_sink.json
  21. 1 1
      desktop/libs/libanalyze/reasons/sort_performance.json
  22. 6 5
      desktop/libs/libanalyze/reasons/spilling.json
  23. 3 2
      desktop/libs/libanalyze/reasons/stats_missing.json
  24. 1 1
      desktop/libs/libanalyze/reasons/too_many_columns.json
  25. 6 0
      desktop/libs/libanalyze/src/libanalyze/analyze.py
  26. 22 4
      desktop/libs/libanalyze/src/libanalyze/models.py
  27. 34 14
      desktop/libs/libanalyze/src/libanalyze/rules.py

+ 1 - 1
apps/impala/src/impala/api.py

@@ -138,9 +138,9 @@ def alanize(request):
     snippets = doc.data_dict.get('snippets', [])
     secret = snippets[0]['result']['handle']['secret']
     impala_query_id = "%x:%x" % struct.unpack(b"QQ", base64.decodestring(secret))
-    api.kill(impala_query_id) # There are many statistics that are not present when the query is open. Close it first.
     query_profile = api.get_query_profile_encoded(impala_query_id)
     profile = analyzer.analyze(analyzer.parse_data(query_profile))
+    ANALYZER.pre_process(profile)
     result = ANALYZER.run(profile)
 
     heatmap = {}

+ 15 - 3
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -179,13 +179,25 @@ class QueryApi(Api):
     query_profile = self.api.get_query_profile_encoded(appid)
     profile = analyzer.analyze(analyzer.parse_data(query_profile))
     ANALYZER.pre_process(profile)
-    return analyzer.metrics(profile)
+    metrics = analyzer.metrics(profile)
+
+    result = ANALYZER.run(profile)
+    if result and result[0]:
+      for factor in result[0]['result']:
+        if factor['reason'] and factor['result_id'] and metrics['nodes'].get(factor['result_id']):
+          metrics['nodes'][factor['result_id']]['health'] = factor['reason']
+    return metrics
 
   def _query(self, appid):
     query = self.api.get_query(query_id=appid)
     query['summary'] = query.get('summary').strip() if query.get('summary') else ''
     query['plan'] = query.get('plan').strip() if query.get('plan') else ''
-    query['metrics'] = self._metrics(appid)
+    try:
+      query['metrics'] = self._metrics(appid)
+    except Exception, e:
+      query['metrics'] = {'nodes' : {}}
+      LOG.exception('Could not parse profile: %s' % e)
+
     if query.get('plan_json'):
       def get_exchange_icon (o):
         if re.search(r'broadcast', o['label_detail'], re.IGNORECASE):
@@ -196,7 +208,7 @@ class QueryApi(Api):
           return { 'svg': 'hi-exchange' }
       def get_sigma_icon (o):
         if re.search(r'streaming', o['label_detail'], re.IGNORECASE):
-          return { 'svg': 'hi-sigma-stream' }
+          return { 'svg': 'hi-sigma' }
         else:
           return { 'svg': 'hi-sigma' }
       mapping = {

Filskillnaden har hållts tillbaka eftersom den är för stor
+ 0 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css


+ 76 - 26
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -300,6 +300,7 @@ function impalaDagre(id) {
     }
     var id = getId(key);
     var localTime = getMaxValue(key, 'LocalTime');
+    localTime = $.extend({}, localTime, { clazz: 'cpu' });
     var last = timeline.filter(function(time) {
       return time.name !== 'Closed';
     }); // Close time is normally wait time;
@@ -312,29 +313,57 @@ function impalaDagre(id) {
     last = last[last.length - 1];
     var time;
     if (!openFinished) {
-      var end = getMetricsMax() || 10;
-      time = { start_time: end - localTime.value, duration: localTime.value, value: end, unit: localTime.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      var end = getMetricsMax() || localTime.value;
+      time = _timelineBefore(end, [localTime]);
     } else if (key.indexOf('EXCHANGE') >= 0 && firstBatchReturned) {
       var triplet = getExchangeCPUIOTimelineData(key);
-      var tripletSum = sum(triplet, 'value');
-      time = [{ start_time: firstBatchReturned.value, duration: triplet[0].value, value: firstBatchReturned.value + triplet[0].value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu }, { start_time: firstBatchReturned.value + triplet[0].value, duration: triplet[1].value, value: firstBatchReturned.value + tripletSum - triplet[2].value, clazz: 'io', unit: last.unit, name: window.HUE_I18n.profile.io }, { start_time: firstBatchReturned.value + tripletSum - triplet[2].value, duration: triplet[2].value, value: firstBatchReturned.value + tripletSum, clazz: 'cpu', unit: last.unit, name: window.HUE_I18n.profile.cpu }]
+      time = _timelineAfter(firstBatchReturned.value, triplet);
     } else if (key.indexOf('JOIN') >= 0) {
       var middle = (openFinished.duration - localTime.value) / 2;
-      time = { start_time: openFinished.start_time + middle, duration: localTime.value, value: openFinished.value - middle, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      var node = getNode(key);
+      var spillTime = getMaxValue(key, 'SpillTime');
+      var joinTimeline;
+      if (spillTime) {
+        joinTimeline = [$.extend({}, spillTime, { clazz: 'io' }), $.extend({}, localTime, {value: localTime.value - spillTime.value})];
+      } else {
+        joinTimeline = [localTime];
+      }
+      time = _timelineAfter(openFinished.start_time + middle, joinTimeline);
     } else if (key.indexOf('UNION') >= 0 || (key.indexOf('AGGREGATE') >= 0 && states_by_name[key].detail.indexOf('STREAMING') >= 0)) {
-      time = { start_time: openFinished.value, duration: localTime.value, value: localTime.value + openFinished.value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      time = _timelineAfter(openFinished.value, [localTime]);
+    } else if (key.indexOf('AGGREGATE') >= 0) {
+      var spillTime = getMaxValue(key, 'SpillTime');
+      var aggTimeline;
+      if (spillTime) {
+        aggTimeline = [$.extend({}, spillTime, { clazz: 'io' }), $.extend({}, localTime, {value: localTime.value - spillTime.value})];
+      } else {
+        aggTimeline = [localTime];
+      }
+      time = _timelineBefore(last.value, aggTimeline);
     } else if (key.indexOf('SCAN') >= 0) {
       var doublet = getScanCPUIOTimelineData(key);
       var doubletSum = sum(doublet, 'value');
       if (firstBatchReturned && firstBatchReturned.start_time + doubletSum < last.value) {
-        time = [{ start_time: firstBatchReturned.start_time, duration: doublet[0].value, value: firstBatchReturned.start_time + doublet[0].value, unit: last.unit, clazz: 'io', name: window.HUE_I18n.profile.io }, { start_time: firstBatchReturned.start_time + doublet[0].value, duration: doublet[1].value, value: firstBatchReturned.start_time + doubletSum, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu }];
+        time = _timelineAfter(firstBatchReturned.start_time, doublet);
       } else {
-        time = [{ start_time: last.value - doubletSum, duration: doublet[0].value, value: last.value - doublet[1].value, unit: last.unit, clazz: 'io', name: window.HUE_I18n.profile.io }, { start_time: last.value - doublet[1].value, duration: doublet[1].value, value: last.value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu }];
+        time = _timelineBefore(last.value, doublet);
       }
     } else {
-      time = { start_time: last.value - localTime.value, duration: localTime.value, value: last.value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      time = _timelineBefore(last.value, [localTime]);
     }
-    return time.length ? time : [ time ];
+    return time;
+  }
+
+  function _timelineBefore(end_time, timeline) {
+    var timelineSum = sum(timeline, 'value');
+    return _timelineAfter(end_time - timelineSum, timeline);
+  }
+
+  function _timelineAfter(start_time, timeline) {
+    var total = 0;
+    return timeline.map(function (event, index) {
+      return $.extend({}, event, { start_time: start_time + total, duration: event.value, value: start_time + (total += event.value), name: window.HUE_I18n.profile[event.clazz]});
+    });
   }
 
   function getExecutionTimelineData(key) {
@@ -346,11 +375,16 @@ function impalaDagre(id) {
     if (!initTime) {
       return timeline;
     }
-    initTime.duration = initTime.value;
-    initTime.name = window.HUE_I18n.profile.codegen;
-    initTime.color = colors[2];
-    initTime.start_time = 0;
-    return [initTime].concat(timeline);
+    return [$.extend({}, initTime, { start_time: 0, duration: initTime.value, name: window.HUE_I18n.profile.codegen, color: colors[2] })].concat(timeline);
+  }
+
+  function getHealthData(key, startTime) {
+    var id = getId(key);
+    var node = getNode(id);
+    if (!node || !node.health) {
+      return;
+    }
+    return _timelineAfter(startTime, node.health.map(function (risk) { return $.extend({}, risk, { value: risk.impact }) }));
   }
 
   function getFragment(id) {
@@ -364,14 +398,21 @@ function impalaDagre(id) {
 
   function getScanCPUIOTimelineData(key) {
     var cpuExchange = getMaxValue(key, 'LocalTime');
+    cpuExchange.clazz = 'cpu';
     var ioTime = getMaxValue(key, 'ChildTime');
+    ioTime.clazz = 'io';
     return [ioTime, cpuExchange];
   }
 
   function getTopNodes() {
     return Object.keys(states_by_name).map(function (key) {
       var timeline = getCPUTimelineData(key);
-      var sumTime = sum(timeline, 'duration')
+      var sumTime;
+      if (timeline) {
+        sumTime = sum(timeline, 'duration');
+      } else {
+        sumTime = states_by_name[key].max_time_val;
+      }
       return { name: states_by_name[key].label, duration: sumTime, unit: 5, key: key, icon: states_by_name[key].icon  };
     }).sort(function (a, b) {
       return b.duration - a.duration;
@@ -383,6 +424,7 @@ function impalaDagre(id) {
     var node = getNode(id);
     var timeline = node.timeline;
     var cpuExchange = getMaxValue(key, 'LocalTime');
+    cpuExchange = $.extend({}, cpuExchange, { clazz: 'cpu' });
 
     var sender = Object.keys(states_by_name).filter(function(node) {
       return states_by_name[node].parent == key;
@@ -391,8 +433,10 @@ function impalaDagre(id) {
       return [{ value: 0, unit: 0 }, { value: 0, unit: 0 }, { value: 0, unit: 0 }];
     }
     var networkTime = getMaxTotalNetworkTime(sender, key);
+    networkTime = $.extend({}, networkTime, { clazz: 'io' });
     var krpcTime = getMaxFragmentMetric(sender, 'LocalTime', 'children.KrpcDataStreamSender.hosts');
-    return [ krpcTime, networkTime, cpuExchange];
+    krpcTime = $.extend({}, krpcTime, { clazz: 'cpu' });
+    return [krpcTime, networkTime, cpuExchange];
   }
 
   function getTimelineData(key, name) {
@@ -435,7 +479,8 @@ function impalaDagre(id) {
       var cpuTimelineSection = detailsContent.append('div').classed('details-section', true);
       var cpuTimelineTitle = cpuTimelineSection.append('header');
       cpuTimelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-filter');
-      cpuTimelineTitle.append('h5').text(window.HUE_I18n.profile.topnodes + ' (' + ko.bindingHandlers.numberFormat.human(getMetricsMax(), 5) + ')');
+      var metricsMax = getMetricsMax() ? ' (' + ko.bindingHandlers.numberFormat.human(getMetricsMax(), 5) + ')' : '';
+      cpuTimelineTitle.append('h5').text(window.HUE_I18n.profile.topnodes + metricsMax);
       var cpuTimelineSectionTable = cpuTimelineSection.append('table').classed('clickable', true);
       var cpuTimelineSectionTableRows = cpuTimelineSectionTable.selectAll('tr').data(topNodes).enter().append('tr').on('click', function (node) {
         select(node.key);
@@ -445,26 +490,26 @@ function impalaDagre(id) {
       cpuTimelineSectionTableRows.append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
     }
 
-    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Compilation'), 'Compilation', null);
-    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Timeline'), window.HUE_I18n.profile.execution, null);
+    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Compilation'), window.HUE_I18n.profile.planning, '#hi-access-time');
+    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Timeline'), window.HUE_I18n.profile.execution, '#hi-access-time');
 
     d3.select('.query-plan .details .metric-title').on('click', function () {
       toggleDetails();
     });
   }
 
-  function appendTimelineAndLegend(detailsContent, data, title, max) {
+  function appendTimelineAndLegend(detailsContent, data, title, icon, max) {
     var timeline = renderTimeline(data, max);
     if (timeline) {
       var executionSum = sum(data, 'duration');
       var cpuTimelineSection = detailsContent.append('div').classed('details-section', true);
       var cpuTimelineTitle = cpuTimelineSection.append('header');
-      cpuTimelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
+      cpuTimelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', icon);
       cpuTimelineTitle.append('h5').text(title + ' (' + ko.bindingHandlers.numberFormat.human(executionSum, 5) + ')');
       cpuTimelineSection.node().appendChild($.parseXML(timeline).children[0]);
 
       var cpuTimelineSectionTable = cpuTimelineSection.append('table').classed('column', true);
-      cpuTimelineSectionTable.append('tr').selectAll('td').data(data).enter().append('td').html(function (time, index) { return '<div class="legend-icon ' + (time.clazz ? time.clazz : '') + '" style="' + (!time.clazz && 'background-color: ' + (time.color || colors[index % colors.length])) + '"></div><div class="metric-name" title="' + time.name + '">' + time.name + '</div>'; });
+      cpuTimelineSectionTable.append('tr').selectAll('td').data(data).enter().append('td').html(function (time, index) { return '<div class="legend-icon ' + (time.clazz ? time.clazz : '') + '" style="' + (!time.clazz && 'background-color: ' + (time.color || colors[index % colors.length])) + '"></div><div class="metric-name" title="' + time.name + (time.message ? ': ' + time.message : '') + '">' + time.name + '</div>'; });
       cpuTimelineSectionTable.append('tr').selectAll('td').data(data).enter().append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
     }
   }
@@ -483,13 +528,18 @@ function impalaDagre(id) {
     details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h4>' + states_by_name[key].label+ '</h4></div>')
     var detailsContent = details.append('div').classed('details-content', true);
 
-    appendTimelineAndLegend(detailsContent, getExecutionTimelineData(key), window.HUE_I18n.profile.execution, getMetricsMax());
-    var timeline = renderTimeline(getTimelineData(key), getMetricsMax());
+    var cpuTimelineData = getCPUTimelineData(key);
+    appendTimelineAndLegend(detailsContent, getExecutionTimelineData(key), window.HUE_I18n.profile.execution, '#hi-microchip', getMetricsMax());
+    appendTimelineAndLegend(detailsContent, getHealthData(key, cpuTimelineData[0] && cpuTimelineData[0].start_time), window.HUE_I18n.profile.risks, '#hi-heart', getMetricsMax());
+
+    var timelineData = getTimelineData(key);
+    var timeline = renderTimeline(timelineData, getMetricsMax());
     if (timeline) {
+      var timelineSum = sum(timelineData, 'duration');
       var timelineSection = detailsContent.append('div').classed('details-section', true);
       var timelineTitle = timelineSection.append('header');
       timelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
-      timelineTitle.append('h5').text(window.HUE_I18n.profile.timeline);
+      timelineTitle.append('h5').text(window.HUE_I18n.profile.timeline + ' (' + ko.bindingHandlers.numberFormat.human(timelineSum, 5) + ')');
       timelineSection.node().appendChild($.parseXML(timeline).children[0]);
 
       var timelineSectionTable = timelineSection.append('table').classed('column', true);

+ 9 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less

@@ -106,6 +106,9 @@
     .name {
       text-transform: capitalize;
       font-size: 13px;
+      width: 115px;
+      overflow: hidden;
+      display: inline-block;
     }
     .detail {
       overflow: hidden;
@@ -291,6 +294,12 @@
           padding-right: 5px;
         }
       }
+      .hi {
+        padding-right: 4px;
+      }
+      .fa {
+        padding-right: 2px;
+      }
       .metrics {
         max-height: calc(~"100% - 40px");
       }

+ 7 - 6
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -7586,7 +7586,7 @@
       humanTime: function (value) {
         value = value * 1;
         if (value < Math.pow(10, 3)) {
-          return value + " ns";
+          return sprintf("%i ns", value);
         } else if (value - Math.pow(10, 6) < -Math.pow(10, 3) / 2) { // Make sure rounding doesn't cause numbers to have more than 4 significant digits.
           value = (value * 1.0) / Math.pow(10, 3);
           var sprint = value > 100 ? "%i us" : "%.1f us";
@@ -7604,17 +7604,18 @@
           var buffer = "";
 
           if (value > (HOUR)) {
-            buffer += sprintf("%i h ", value / HOUR);
+            buffer += sprintf("%i h", value / HOUR);
             value = value % HOUR;
           }
 
-          if (value > MINUTE) {
-            buffer += sprintf("%i m ", value / MINUTE);
+          if (buffer.length < 4 && value > MINUTE) {
+            var sprint = buffer.length ? " %i m" : "%i m";
+            buffer += sprintf(sprint, value / MINUTE);
             value = value % MINUTE;
           }
 
-          if (value > SECOND) {
-            var sprint = buffer.length ? "%i s" : "%.1f s";
+          if (buffer.length < 4 && value > SECOND) {
+            var sprint = buffer.length ? " %i s" : "%.1f s";
             buffer += sprintf(sprint, value * 1.0 / SECOND);
           }
           return buffer;

+ 3 - 1
desktop/core/src/desktop/templates/global_js_constants.mako

@@ -185,7 +185,9 @@
       codegen: "${ _('CodeGen') }",
       overview: "${ _('Overview') }",
       topnodes: "${ _('Top Nodes') }",
-      compilation: "${ _('Compilation') }"
+      compilation: "${ _('Compilation') }",
+      planning: "${ _('Planning') }",
+      risks: "${ _('Risks') }"
     }
   };
 

+ 7 - 0
desktop/core/src/desktop/templates/hue_icons.mako

@@ -384,6 +384,13 @@
     <symbol id="hi-copy" viewBox="0 0 512 512">
       <path d="M433.941 65.941l-51.882-51.882A48 48 0 0 0 348.118 0H176c-26.51 0-48 21.49-48 48v48H48c-26.51 0-48 21.49-48 48v320c0 26.51 21.49 48 48 48h224c26.51 0 48-21.49 48-48v-48h80c26.51 0 48-21.49 48-48V99.882a48 48 0 0 0-14.059-33.941zM266 464H54a6 6 0 0 1-6-6V150a6 6 0 0 1 6-6h74v224c0 26.51 21.49 48 48 48h96v42a6 6 0 0 1-6 6zm128-96H182a6 6 0 0 1-6-6V54a6 6 0 0 1 6-6h106v88c0 13.255 10.745 24 24 24h88v202a6 6 0 0 1-6 6zm6-256h-64V48h9.632c1.591 0 3.117.632 4.243 1.757l48.368 48.368a6 6 0 0 1 1.757 4.243V112z"></path>
     </symbol>
+    <symbol id="hi-microchip" viewBox="0 0 1024 1024">
+      <path d="M682.67 307.2h-341.34c-18.852 0-34.13 15.281-34.13 34.13v341.34c0 18.852 15.28 34.13 34.13 34.13h341.34c18.852 0 34.13-15.28 34.13-34.13v-341.34c0-18.851-15.28-34.13-34.13-34.13z" fill="" />
+      <path d="M921.606 546.133c18.85 0 34.127-15.282 34.127-34.133 0-18.855-15.279-34.133-34.127-34.133h-68.273v-102.4h68.273c18.85 0 34.127-15.282 34.127-34.133 0-18.855-15.279-34.133-34.127-34.133h-68.273v-68.267c0-37.706-30.564-68.267-68.267-68.267h-68.267v-68.273c0-18.85-15.282-34.127-34.133-34.127-18.855 0-34.133 15.279-34.133 34.127v68.273h-102.4v-68.273c0-18.85-15.282-34.127-34.133-34.127-18.855 0-34.133 15.279-34.133 34.127v68.273h-102.4v-68.273c0-18.85-15.282-34.127-34.133-34.127-18.855 0-34.133 15.279-34.133 34.127v68.273h-68.267c-37.706 0-68.267 30.564-68.267 68.267v68.267h-68.273c-18.85 0-34.127 15.282-34.127 34.133 0 18.855 15.279 34.133 34.127 34.133h68.273v102.4h-68.273c-18.85 0-34.127 15.282-34.127 34.133 0 18.855 15.279 34.133 34.127 34.133h68.273v102.4h-68.273c-18.85 0-34.127 15.282-34.127 34.133 0 18.855 15.279 34.133 34.127 34.133h68.273v68.267c0 37.706 30.564 68.267 68.267 68.267h68.267v68.273c0 18.85 15.282 34.127 34.133 34.127 18.855 0 34.133-15.279 34.133-34.127v-68.273h102.4v68.273c0 18.85 15.282 34.127 34.133 34.127 18.855 0 34.133-15.279 34.133-34.127v-68.273h102.4v68.273c0 18.85 15.282 34.127 34.133 34.127 18.855 0 34.133-15.279 34.133-34.127v-68.273h68.267c37.706 0 68.267-30.564 68.267-68.267v-68.267h68.273c18.85 0 34.127-15.282 34.127-34.133 0-18.855-15.279-34.133-34.127-34.133h-68.273v-102.4h68.273zM785.067 785.067h-511.996c-18.853 0-34.138-15.284-34.138-34.138v-511.996h511.996c18.854 0 34.138 15.284 34.138 34.138v511.996z" fill="" />
+    </symbol>
+    <symbol id="hi-heart" viewBox="0 0 1024 1024">
+      <path d="M886.250667 552.490667 512 927.957333l-374.229333-375.466667C79.786667 505.429333 42.666667 433.536 42.666667 352.896 42.666667 211.029333 157.290667 96.042667 298.666667 96.042667c89.088 0 167.488 45.717333 213.333333 114.986667 45.845333-69.269333 124.245333-114.986667 213.333333-114.986667 141.376 0 256 114.986667 256 256.832C981.333333 433.536 944.213333 505.429333 886.250667 552.490667zM832.341333 458.858667l-138.858667 0-53.888-197.610667-20.586667 5.610667-20.117333-7.104-93.226667 317.034667-101.077333-252.672-20.245333 6.741333-19.989333-7.509333-50.794667 135.509333L192.341333 458.858667l0 42.666667 128 0 19.989333 7.509333 44.672-119.168 107.114667 267.754667 40.469333-13.504-3.477333-8.661333 3.349333 1.173333 85.76-291.562667 44.202667 162.090667 20.586667-5.610667 149.333333 0L832.341333 458.858667z" />
+    </symbol>
   </svg>
 
   <script type="text/html" id="app-switcher-icon-template">

+ 1 - 0
desktop/libs/libanalyze/reasons/agg_performance.json

@@ -4,6 +4,7 @@
   "metric_names": [
     "LocalTime"
   ],
+  "unit": 5,
   "rule": {
     "expr": "vars['LocalTime'] - float(vars['InputRows']) / 0.01",
     "message": "Excess time (over expected time) spent in the aggregate; might be caused by complex group by",

+ 1 - 1
desktop/libs/libanalyze/reasons/bytes_read_skew.json

@@ -8,7 +8,7 @@
     "max",
     "avg"
   ],
-  "unit_id": 0,
+  "unit": 5,
   "rule": {
     "condition": "vars['IOBound']==True",
     "expr": "(vars['max'] - vars['avg']) / 100000000 / 5",

+ 1 - 0
desktop/libs/libanalyze/reasons/join_performance.json

@@ -5,6 +5,7 @@
     "ProbeRows",
     "ProbeTime"
   ],
+  "unit": 5,
   "rule": {
     "expr": "vars['ProbeTime'] - float(vars['ProbeRows']) / 0.005",
     "message": "Excess time (over expected time) spent in the hash join",

+ 1 - 1
desktop/libs/libanalyze/reasons/metadata_missing.json

@@ -5,7 +5,7 @@
     "MetadataLoadTime",
     "PlanningTime"
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
     "condition": "float(vars['MetadataLoadTime']) / float(vars['PlanningTime']) > 0.5 and vars['MetadataLoadTime'] > 2000000000",
     "expr": "vars['MetadataLoadTime']",

+ 1 - 1
desktop/libs/libanalyze/reasons/remote_scan_ranges.json

@@ -4,7 +4,7 @@
   "metric_names": [
     "BytesReadRemoteUnexpected"
   ],
-  "unit_id": 0,
+  "unit": 5,
   "rule": {
     "condition": "vars['IOBound']==True",
     "expr": "vars['BytesReadRemoteUnexpected'] * (1/30 - 1/100) * 1/1024/1024",

+ 1 - 1
desktop/libs/libanalyze/reasons/rows_read_skew.json

@@ -8,7 +8,7 @@
     "max",
     "avg"
   ],
-  "unit_id": 0,
+  "unit": 5,
   "rule": {
     "expr": "(vars['max'] - vars['avg']) / 40.0 * 1000",
     "message": "The skew (max-avg) in rows processed",

+ 1 - 1
desktop/libs/libanalyze/reasons/scan_performance.json

@@ -7,7 +7,7 @@
     "ScannerThreadsSysTime",
     "AverageScannerThreadConcurrency"
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
     "expr": "(vars['ScannerThreadsUserTime'] + vars['ScannerThreadsSysTime'] - vars['RowsRead'] * 100) / max(1,vars['AverageScannerThreadConcurrency'])",
     "message": "Predicates might be expensive (expectes speed 10m rows per sec per core)",

+ 2 - 1
desktop/libs/libanalyze/reasons/scanner_filter.json

@@ -4,9 +4,10 @@
   "metric_names": [
     "RowsRead", "RowsReturned"
   ],
+  "unit": 5,
   "rule": {
     "condition": "vars['RowsReturned'] < vars['RowsRead']",
-    "expr": "vars['RowsRead'] - vars['RowsReturned']",
+    "expr": "(vars['RowsRead'] - vars['RowsReturned']) / 40.0 * 1000",
     "message": "Kudu could not evaluate the predicate and it was evaluated by Impala. Kudu supports predicates =, <=, <, >, >=, BETWEEN, or IN.",
     "label": "Kudu predicate optimization"
   },

+ 1 - 1
desktop/libs/libanalyze/reasons/scanner_parallelism.json

@@ -5,7 +5,7 @@
     "AverageScannerThreadConcurrency",
     "LocalTime"
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
     "condition": "vars['IOBound']==False",
     "expr": "float(8 - vars['AverageScannerThreadConcurrency']) / 8.0 * vars['LocalTime']",

+ 1 - 0
desktop/libs/libanalyze/reasons/selective_scan.json

@@ -6,6 +6,7 @@
     "RowsReturned",
     "LocalTime"
   ],
+  "unit": 5,
   "rule": {
     "condition": "vars['RowsRead']>8000000",
     "expr": "float(vars['RowsRead'] - vars['RowsReturned']) / vars['LocalTime']",

+ 1 - 1
desktop/libs/libanalyze/reasons/skew.json

@@ -6,7 +6,7 @@
     "max",
     "avg"
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
     "message": "The skew (max-avg) contributed this amount of time to this SQL operator",
     "expr": "(vars['max'] - vars['avg'])",

+ 1 - 1
desktop/libs/libanalyze/reasons/slow_table_sink.json

@@ -6,7 +6,7 @@
     "BytesWritten",
     "LocalTime"
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
     "condition": "vars['BytesWritten'] > 0",
     "expr": "float(vars['LocalTime']) - 0.01 / float(vars['BytesWritten'])",

+ 1 - 1
desktop/libs/libanalyze/reasons/sort_performance.json

@@ -4,7 +4,7 @@
   "metric_names": [
     "LocalTime"
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
     "expr": "vars['LocalTime'] - float(vars['InputRows']) / 0.01",
     "message": "Excess time (over expected time) spent in the sort; might be caused by too many sorting column",

+ 6 - 5
desktop/libs/libanalyze/reasons/spilling.json

@@ -1,11 +1,12 @@
 {
   "type": "SQLOperator",
-  "node_name": ["HashJoinNode", "AGGREGATION_NODE"],
-  "metric_names": "SpilledPartitions",
+  "node_name": ["HASH_JOIN_NODE", "AGGREGATION_NODE"],
+  "metric_names": "SpillTime",
+  "unit": 5,
   "rule": {
-    "condition": "vars['SpilledPartitions'] > 0",
-    "expr": "1",
-    "message": "This operation has spilled to disk. Check if the ressource configuration of Impala can be changed to allow for a higher memory limit.",
+    "condition": "vars['SpillTime'] > 0",
+    "expr": "vars['SpillTime']",
+    "message": "This operation has spilled to disk. Check if the resource configuration of Impala can be changed to allow for a higher memory limit.",
     "label": " Spilled Partitions"
   },
   "fix": {

+ 3 - 2
desktop/libs/libanalyze/reasons/stats_missing.json

@@ -2,12 +2,13 @@
   "type": "SQLOperator",
   "node_name": ["HDFS_SCAN_NODE", "KUDU_SCAN_NODE", "HBASE_SCAN_NODE"],
   "metric_names": [
-    "MissingStats"
+    "MissingStats", "TotalTime"
   ],
   "info_names": ["Table"],
+  "unit": 5,
   "rule": {
     "condition": "vars['MissingStats'] == 1",
-    "expr": "1",
+    "expr": "vars['TotalTime'] * 0.2",
     "message": "The statistics are missing or corrupt which prevent scan optimizations.",
     "label": "Statistics Missing"
   },

+ 1 - 1
desktop/libs/libanalyze/reasons/too_many_columns.json

@@ -6,7 +6,7 @@
     "NumColumns",
     "LocalTime"
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
     "condition": "vars['NumColumns'] > 15",
     "expr": "float(vars['NumColumns'] - 15) / float(vars['NumColumns']) * vars['LocalTime']",

+ 6 - 0
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -317,15 +317,18 @@ def metrics(profile):
   def flatten(node, counter_map=counter_map):
     is_plan_node = node.is_plan_node()
     is_parent_node = is_plan_node
+    bIncludeInMinMax = True
     if not is_plan_node:
       if node.plan_node:
         nid = node.plan_node.id()
       elif node.is_fragment_instance():
         is_parent_node = True
         nid = node.fragment.id()
+        bIncludeInMinMax = False
       elif node.is_fragment() and node.is_averaged():
         is_parent_node = True
         nid = node.id()
+        bIncludeInMinMax = False
       elif node.fragment:
         nid = node.fragment.id()
       else:
@@ -348,6 +351,7 @@ def metrics(profile):
         counter_map['nodes'][nid]['other'] = plan_json[nid]
       if is_plan_node:
         counter_map['nodes'][nid]['fragment'] = node.fragment.id()
+      counter_map['nodes'][nid]['timeline']['minmax'] = bIncludeInMinMax
     else:
       name = node.name()
       if counter_map['nodes'][nid]['children'].get(name) is None:
@@ -359,6 +363,8 @@ def metrics(profile):
   for nodeid, node in counter_map['nodes'].iteritems():
     host_min = {'value': sys.maxint, 'host' : None}
     host_max = {'value': -(sys.maxint - 1), 'host' : None}
+    if not node['timeline']['minmax']:
+      continue
     for host_name, host_value in node['timeline']['hosts'].iteritems():
       for event_name, event in host_value.iteritems():
         if len(event):

+ 22 - 4
desktop/libs/libanalyze/src/libanalyze/models.py

@@ -33,6 +33,7 @@ class Contributor(object):
 
 class Reason(object):
   def __init__(self, **kwargs):
+    self.name = None
     self.message = None
     self.impact = None
     self.unit = None
@@ -56,11 +57,28 @@ def query_node_by_id(profile, node_id, metric_name, averaged=False):
   result = profile.find_by_id(node_id)
   if not result:
     return result
-  nodes = filter(lambda x: x.fragment.is_averaged() == averaged, result)
+
+  nodes = _filter_averaged(result, averaged)
   metric = reduce(lambda x, y: x + y.find_metric_by_name(metric_name), nodes, [])
 
   return map(lambda x: L(x['value'], x['unit'], 0, x['node'].fragment.id(), x['node'].host(), 0, x['node'].id(), x['node'].name(), value=x['value'], unit=x['unit'], fragment_id=0, fid=x['node'].fragment.id(), host=x['node'].host(), node_id=x['node'].id(), name=x['node'].name(), node=x['node']), metric)
 
+def _filter_averaged(result, averaged=False):
+  #nodes = filter(lambda x: x.fragment.is_averaged() == averaged, result)
+  # Averaged results are not always present. If we're looking for averaged results, sort by averaged and get first result (hopefully getting averaged!).
+  # If we're not looking for averaged results, remove them.
+  if averaged:
+    def by_averaged(x, y):
+      if x.fragment.is_averaged():
+        return -1
+      elif y.fragment.is_averaged():
+        return 1
+      else:
+        return 0
+    return sorted(result, cmp=by_averaged)
+  else:
+    return filter(lambda x: x.fragment.is_averaged() == averaged, result)
+
 def query_node_by_metric(profile, node_name, metric_name):
   """Given the query_id, searches for the corresponding query profile and
   selects the node instances given by node_name, selects the metric given by
@@ -91,7 +109,7 @@ def query_element_by_info(profile, node_name, metric_name):
   metric = reduce(lambda x, y: x + y.find_info_by_name(metric_name), nodes, [])
   return map(lambda x: L(x['value'], 0, x['node'].fragment.id() if x['node'].fragment else '', x['node'].host(), 0, x['node'].id(), x['node'].name(), value=x['value'], fragment_id=0, fid=x['node'].fragment.id() if x['node'].fragment else '', host=x['node'].host(), node_id=x['node'].id(), name=x['node'].name(), node=x['node']), metric)
 
-def query_avg_fragment_metric_by_node_nid(profile, node_nid, metric_name):
+def query_avg_fragment_metric_by_node_nid(profile, node_nid, metric_name, default):
   """
   Given the surragate node id (i.e. unique id of the plan node in the database),
   return the value of the fragment level metric.
@@ -102,9 +120,9 @@ def query_avg_fragment_metric_by_node_nid(profile, node_nid, metric_name):
   result = profile.find_by_id(node_nid)
   if not result:
     return result
-  node = map(lambda x: x, filter(lambda x: x.fragment.is_averaged() == True, result))[0]
+  node = _filter_averaged(result, True)[0]
   metric = node.fragment.find_metric_by_name(metric_name)
-  return metric[0]['value']
+  return metric and metric[0]['value'] or default
 
 def query_fragment_metric_by_node_id(node, metric_name):
   """

+ 34 - 14
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -170,10 +170,10 @@ class SQLOperatorReason:
                 local_vars['vars'].update(dict(zip(self.kwargs['info_names'], metric_values)))
                 expr_data = exprs.Expr.evaluate(self.kwargs['fix']['data'], local_vars)
 
-        msg = self.rule["label"] + ": " + self.rule["message"]
         return {
             "impact": impact,
-            "message": msg,
+            "message": self.rule["message"],
+            "label": self.rule["label"],
             "data": expr_data
         }
 
@@ -238,10 +238,10 @@ class SummaryReason(SQLOperatorReason):
                     if (impact is None or impact < expr_val):
                         impact = expr_val
 
-        msg = self.rule["label"] + ": " + self.rule["message"]
         return {
             "impact": impact,
-            "message": msg
+            "message": self.rule["message"],
+            "label": self.rule["label"]
         }
 
 class JoinOrderStrategyCheck(SQLOperatorReason):
@@ -282,7 +282,8 @@ class JoinOrderStrategyCheck(SQLOperatorReason):
         if (impact > 0):
             return {
                 "impact": impact,
-                "message": "Wrong join order - RHS %d; LHS %d" % (rhsRows, lhsRows)
+                "message": "RHS %d; LHS %d" % (rhsRows, lhsRows),
+                "label": "Wrong join order"
             }
 
         bcost = rhsRows * hosts
@@ -290,7 +291,8 @@ class JoinOrderStrategyCheck(SQLOperatorReason):
         impact = (networkcost - min(bcost, scost) - 1) / hosts / 0.01
         return {
             "impact": impact,
-            "message": "Wrong join strategy - RHS %d; LHS %d" % (rhsRows, lhsRows)
+            "message": "RHS %d; LHS %d" % (rhsRows, lhsRows),
+            "label": "Wrong join strategy"
         }
 
 class ExplodingJoinCheck(SQLOperatorReason):
@@ -319,7 +321,8 @@ class ExplodingJoinCheck(SQLOperatorReason):
             impact = probeTime * (rowsReturned - probeRows) / rowsReturned
         return {
             "impact": impact,
-            "message": "Exploding join: %d input rows are exploded to %d output rows" % (probeRows, rowsReturned)
+            "message": "%d input rows are exploded to %d output rows" % (probeRows, rowsReturned),
+            "label": "Exploding join"
         }
 
 class NNRpcCheck(SQLOperatorReason):
@@ -336,14 +339,15 @@ class NNRpcCheck(SQLOperatorReason):
         }
         :return:
         """
-        totalStorageTime = models.query_avg_fragment_metric_by_node_nid(profile, plan_node_id, "TotalStorageWaitTime")
+        totalStorageTime = models.query_avg_fragment_metric_by_node_nid(profile, plan_node_id, "TotalStorageWaitTime", 0)
         hdfsRawReadTime = models.query_node_by_id(profile, plan_node_id, "TotalRawHdfsReadTime(*)", True)[0][0]
         avgReadThreads = models.query_node_by_id(profile, plan_node_id, "AverageHdfsReadThreadConcurrency", True)[0][0]
         avgReadThreads = max(1, to_double(avgReadThreads))
         impact = max(0, (totalStorageTime - hdfsRawReadTime) / avgReadThreads)
         return {
             "impact": impact,
-            "message": "This is the time waiting for HDFS NN RPC."
+            "message": "This is the time waiting for HDFS NN RPC.",
+            "label": "HDFS NN RPC"
         }
 
 class TopDownAnalysis:
@@ -488,7 +492,7 @@ class TopDownAnalysis:
                 fix.update(cause.kwargs['fix'])
                 if evaluation.get('data'):
                   fix['data'] = evaluation['data']
-                reason = models.Reason(message=evaluation['message'], impact=evaluation['impact'], unit=cause.kwargs.get('unit_id', ''), fix=fix)
+                reason = models.Reason(name=evaluation['label'], message=evaluation['message'], impact=evaluation['impact'], unit=cause.kwargs.get('unit', ''), fix=fix)
                 reasons.append(reason)
         return sorted(reasons, key=lambda x: x.impact, reverse=True)
 
@@ -614,7 +618,8 @@ class TopDownAnalysis:
             counter_map = node.counter_map()
 
             # Load the metric data as if the object would be loaded from the DB
-            local_time = counter_map['TotalTime'].value - child_time
+            local_time = max(counter_map['TotalTime'].value - child_time, 0)
+            has_spilled = False
 
             # Make sure to substract the wait time for the exchange node
             if is_plan_node and re.search(r'EXCHANGE_NODE', node.val.name) is not None:
@@ -634,14 +639,24 @@ class TopDownAnalysis:
             if re.search(r'KUDU_SCAN_NODE', node.val.name):
               child_time = counter_map.get('KuduClientTime', models.TCounter(value=0)).value
               local_time = counter_map['TotalTime'].value
-              counter_map['TotalTime'].value = child_time + local_time
             if re.search(r'HDFS_SCAN_NODE', node.val.name):
               child_time = counter_map.get('TotalRawHdfsReadTime(*)', models.TCounter(value=0)).value
               local_time = counter_map['TotalTime'].value
-              counter_map['TotalTime'].value = local_time + child_time
+            if re.search(r'Buffer pool', node.val.name):
+              local_time = counter_map.get('WriteIoWaitTime', models.TCounter(value=0)).value + counter_map.get('ReadIoWaitTime', models.TCounter(value=0)).value + counter_map.get('AllocTime', models.TCounter(value=0)).value
+            if counter_map.get('SpilledPartitions', 0) > 0:
+              has_spilled = True
+
+            if re.search(r'AGGREGATION', node.val.name):
+              grouping_aggregator = node.find_by_name('GroupingAggregator')
+              if grouping_aggregator and grouping_aggregator.counter_map().get('SpilledPartitions', models.TCounter(value=0)).value > 0:
+                has_spilled = True
 
             # For Hash Join, if the "LocalTime" metrics
             if is_plan_node and re.search(r'HASH_JOIN_NODE', node.val.name) is not None:
+                hash_join_builder = node.find_by_name('Hash Join Builder')
+                if hash_join_builder and hash_join_builder.counter_map().get('SpilledPartitions', models.TCounter(value=0)).value > 0:
+                  has_spilled = True
                 if ("LocalTime" in counter_map):
                     local_time = counter_map["LocalTime"].value
                 else:
@@ -649,13 +664,18 @@ class TopDownAnalysis:
                         counter_map["BuildTime"].value
 
             # Add two virtual metrics for local_time and child_time
+            if has_spilled:
+              spill_time = 0
+              buffer_pool = node.find_by_name('Buffer pool')
+              if buffer_pool:
+                spill_time = buffer_pool.counter_map()['LocalTime'].value
+              node.val.counters.append(models.TCounter(name='SpillTime', value=spill_time, unit=5))
             node.val.counters.append(models.TCounter(name='LocalTime', value=local_time, unit=5))
             node.val.counters.append(models.TCounter(name='ChildTime', value=child_time, unit=5))
 
         profile.foreach_lambda(add_host)
 
     def run(self, profile):
-        self.pre_process(profile)
         contributors = self.process(profile)
         topContributors = self.getTopContributor(100, contributors)
 

Vissa filer visades inte eftersom för många filer har ändrats