Browse Source

HUE-8704 [jb] Add health risks to profile.

jdesjean 6 years ago
parent
commit
96fbe70bd4
27 changed files with 204 additions and 73 deletions
  1. 1 1
      apps/impala/src/impala/api.py
  2. 15 3
      apps/jobbrowser/src/jobbrowser/apis/query_api.py
  3. 0 0
      apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css
  4. 76 26
      apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js
  5. 9 0
      apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less
  6. 7 6
      desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js
  7. 3 1
      desktop/core/src/desktop/templates/global_js_constants.mako
  8. 7 0
      desktop/core/src/desktop/templates/hue_icons.mako
  9. 1 0
      desktop/libs/libanalyze/reasons/agg_performance.json
  10. 1 1
      desktop/libs/libanalyze/reasons/bytes_read_skew.json
  11. 1 0
      desktop/libs/libanalyze/reasons/join_performance.json
  12. 1 1
      desktop/libs/libanalyze/reasons/metadata_missing.json
  13. 1 1
      desktop/libs/libanalyze/reasons/remote_scan_ranges.json
  14. 1 1
      desktop/libs/libanalyze/reasons/rows_read_skew.json
  15. 1 1
      desktop/libs/libanalyze/reasons/scan_performance.json
  16. 2 1
      desktop/libs/libanalyze/reasons/scanner_filter.json
  17. 1 1
      desktop/libs/libanalyze/reasons/scanner_parallelism.json
  18. 1 0
      desktop/libs/libanalyze/reasons/selective_scan.json
  19. 1 1
      desktop/libs/libanalyze/reasons/skew.json
  20. 1 1
      desktop/libs/libanalyze/reasons/slow_table_sink.json
  21. 1 1
      desktop/libs/libanalyze/reasons/sort_performance.json
  22. 6 5
      desktop/libs/libanalyze/reasons/spilling.json
  23. 3 2
      desktop/libs/libanalyze/reasons/stats_missing.json
  24. 1 1
      desktop/libs/libanalyze/reasons/too_many_columns.json
  25. 6 0
      desktop/libs/libanalyze/src/libanalyze/analyze.py
  26. 22 4
      desktop/libs/libanalyze/src/libanalyze/models.py
  27. 34 14
      desktop/libs/libanalyze/src/libanalyze/rules.py

+ 1 - 1
apps/impala/src/impala/api.py

@@ -138,9 +138,9 @@ def alanize(request):
     snippets = doc.data_dict.get('snippets', [])
     snippets = doc.data_dict.get('snippets', [])
     secret = snippets[0]['result']['handle']['secret']
     secret = snippets[0]['result']['handle']['secret']
     impala_query_id = "%x:%x" % struct.unpack(b"QQ", base64.decodestring(secret))
     impala_query_id = "%x:%x" % struct.unpack(b"QQ", base64.decodestring(secret))
-    api.kill(impala_query_id) # There are many statistics that are not present when the query is open. Close it first.
     query_profile = api.get_query_profile_encoded(impala_query_id)
     query_profile = api.get_query_profile_encoded(impala_query_id)
     profile = analyzer.analyze(analyzer.parse_data(query_profile))
     profile = analyzer.analyze(analyzer.parse_data(query_profile))
+    ANALYZER.pre_process(profile)
     result = ANALYZER.run(profile)
     result = ANALYZER.run(profile)
 
 
     heatmap = {}
     heatmap = {}

+ 15 - 3
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -179,13 +179,25 @@ class QueryApi(Api):
     query_profile = self.api.get_query_profile_encoded(appid)
     query_profile = self.api.get_query_profile_encoded(appid)
     profile = analyzer.analyze(analyzer.parse_data(query_profile))
     profile = analyzer.analyze(analyzer.parse_data(query_profile))
     ANALYZER.pre_process(profile)
     ANALYZER.pre_process(profile)
-    return analyzer.metrics(profile)
+    metrics = analyzer.metrics(profile)
+
+    result = ANALYZER.run(profile)
+    if result and result[0]:
+      for factor in result[0]['result']:
+        if factor['reason'] and factor['result_id'] and metrics['nodes'].get(factor['result_id']):
+          metrics['nodes'][factor['result_id']]['health'] = factor['reason']
+    return metrics
 
 
   def _query(self, appid):
   def _query(self, appid):
     query = self.api.get_query(query_id=appid)
     query = self.api.get_query(query_id=appid)
     query['summary'] = query.get('summary').strip() if query.get('summary') else ''
     query['summary'] = query.get('summary').strip() if query.get('summary') else ''
     query['plan'] = query.get('plan').strip() if query.get('plan') else ''
     query['plan'] = query.get('plan').strip() if query.get('plan') else ''
-    query['metrics'] = self._metrics(appid)
+    try:
+      query['metrics'] = self._metrics(appid)
+    except Exception, e:
+      query['metrics'] = {'nodes' : {}}
+      LOG.exception('Could not parse profile: %s' % e)
+
     if query.get('plan_json'):
     if query.get('plan_json'):
       def get_exchange_icon (o):
       def get_exchange_icon (o):
         if re.search(r'broadcast', o['label_detail'], re.IGNORECASE):
         if re.search(r'broadcast', o['label_detail'], re.IGNORECASE):
@@ -196,7 +208,7 @@ class QueryApi(Api):
           return { 'svg': 'hi-exchange' }
           return { 'svg': 'hi-exchange' }
       def get_sigma_icon (o):
       def get_sigma_icon (o):
         if re.search(r'streaming', o['label_detail'], re.IGNORECASE):
         if re.search(r'streaming', o['label_detail'], re.IGNORECASE):
-          return { 'svg': 'hi-sigma-stream' }
+          return { 'svg': 'hi-sigma' }
         else:
         else:
           return { 'svg': 'hi-sigma' }
           return { 'svg': 'hi-sigma' }
       mapping = {
       mapping = {

File diff suppressed because it is too large
+ 0 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css


+ 76 - 26
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -300,6 +300,7 @@ function impalaDagre(id) {
     }
     }
     var id = getId(key);
     var id = getId(key);
     var localTime = getMaxValue(key, 'LocalTime');
     var localTime = getMaxValue(key, 'LocalTime');
+    localTime = $.extend({}, localTime, { clazz: 'cpu' });
     var last = timeline.filter(function(time) {
     var last = timeline.filter(function(time) {
       return time.name !== 'Closed';
       return time.name !== 'Closed';
     }); // Close time is normally wait time;
     }); // Close time is normally wait time;
@@ -312,29 +313,57 @@ function impalaDagre(id) {
     last = last[last.length - 1];
     last = last[last.length - 1];
     var time;
     var time;
     if (!openFinished) {
     if (!openFinished) {
-      var end = getMetricsMax() || 10;
-      time = { start_time: end - localTime.value, duration: localTime.value, value: end, unit: localTime.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      var end = getMetricsMax() || localTime.value;
+      time = _timelineBefore(end, [localTime]);
     } else if (key.indexOf('EXCHANGE') >= 0 && firstBatchReturned) {
     } else if (key.indexOf('EXCHANGE') >= 0 && firstBatchReturned) {
       var triplet = getExchangeCPUIOTimelineData(key);
       var triplet = getExchangeCPUIOTimelineData(key);
-      var tripletSum = sum(triplet, 'value');
-      time = [{ start_time: firstBatchReturned.value, duration: triplet[0].value, value: firstBatchReturned.value + triplet[0].value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu }, { start_time: firstBatchReturned.value + triplet[0].value, duration: triplet[1].value, value: firstBatchReturned.value + tripletSum - triplet[2].value, clazz: 'io', unit: last.unit, name: window.HUE_I18n.profile.io }, { start_time: firstBatchReturned.value + tripletSum - triplet[2].value, duration: triplet[2].value, value: firstBatchReturned.value + tripletSum, clazz: 'cpu', unit: last.unit, name: window.HUE_I18n.profile.cpu }]
+      time = _timelineAfter(firstBatchReturned.value, triplet);
     } else if (key.indexOf('JOIN') >= 0) {
     } else if (key.indexOf('JOIN') >= 0) {
       var middle = (openFinished.duration - localTime.value) / 2;
       var middle = (openFinished.duration - localTime.value) / 2;
-      time = { start_time: openFinished.start_time + middle, duration: localTime.value, value: openFinished.value - middle, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      var node = getNode(key);
+      var spillTime = getMaxValue(key, 'SpillTime');
+      var joinTimeline;
+      if (spillTime) {
+        joinTimeline = [$.extend({}, spillTime, { clazz: 'io' }), $.extend({}, localTime, {value: localTime.value - spillTime.value})];
+      } else {
+        joinTimeline = [localTime];
+      }
+      time = _timelineAfter(openFinished.start_time + middle, joinTimeline);
     } else if (key.indexOf('UNION') >= 0 || (key.indexOf('AGGREGATE') >= 0 && states_by_name[key].detail.indexOf('STREAMING') >= 0)) {
     } else if (key.indexOf('UNION') >= 0 || (key.indexOf('AGGREGATE') >= 0 && states_by_name[key].detail.indexOf('STREAMING') >= 0)) {
-      time = { start_time: openFinished.value, duration: localTime.value, value: localTime.value + openFinished.value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      time = _timelineAfter(openFinished.value, [localTime]);
+    } else if (key.indexOf('AGGREGATE') >= 0) {
+      var spillTime = getMaxValue(key, 'SpillTime');
+      var aggTimeline;
+      if (spillTime) {
+        aggTimeline = [$.extend({}, spillTime, { clazz: 'io' }), $.extend({}, localTime, {value: localTime.value - spillTime.value})];
+      } else {
+        aggTimeline = [localTime];
+      }
+      time = _timelineBefore(last.value, aggTimeline);
     } else if (key.indexOf('SCAN') >= 0) {
     } else if (key.indexOf('SCAN') >= 0) {
       var doublet = getScanCPUIOTimelineData(key);
       var doublet = getScanCPUIOTimelineData(key);
       var doubletSum = sum(doublet, 'value');
       var doubletSum = sum(doublet, 'value');
       if (firstBatchReturned && firstBatchReturned.start_time + doubletSum < last.value) {
       if (firstBatchReturned && firstBatchReturned.start_time + doubletSum < last.value) {
-        time = [{ start_time: firstBatchReturned.start_time, duration: doublet[0].value, value: firstBatchReturned.start_time + doublet[0].value, unit: last.unit, clazz: 'io', name: window.HUE_I18n.profile.io }, { start_time: firstBatchReturned.start_time + doublet[0].value, duration: doublet[1].value, value: firstBatchReturned.start_time + doubletSum, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu }];
+        time = _timelineAfter(firstBatchReturned.start_time, doublet);
       } else {
       } else {
-        time = [{ start_time: last.value - doubletSum, duration: doublet[0].value, value: last.value - doublet[1].value, unit: last.unit, clazz: 'io', name: window.HUE_I18n.profile.io }, { start_time: last.value - doublet[1].value, duration: doublet[1].value, value: last.value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu }];
+        time = _timelineBefore(last.value, doublet);
       }
       }
     } else {
     } else {
-      time = { start_time: last.value - localTime.value, duration: localTime.value, value: last.value, unit: last.unit, clazz: 'cpu', name: window.HUE_I18n.profile.cpu };
+      time = _timelineBefore(last.value, [localTime]);
     }
     }
-    return time.length ? time : [ time ];
+    return time;
+  }
+
+  function _timelineBefore(end_time, timeline) {
+    var timelineSum = sum(timeline, 'value');
+    return _timelineAfter(end_time - timelineSum, timeline);
+  }
+
+  function _timelineAfter(start_time, timeline) {
+    var total = 0;
+    return timeline.map(function (event, index) {
+      return $.extend({}, event, { start_time: start_time + total, duration: event.value, value: start_time + (total += event.value), name: window.HUE_I18n.profile[event.clazz]});
+    });
   }
   }
 
 
   function getExecutionTimelineData(key) {
   function getExecutionTimelineData(key) {
@@ -346,11 +375,16 @@ function impalaDagre(id) {
     if (!initTime) {
     if (!initTime) {
       return timeline;
       return timeline;
     }
     }
-    initTime.duration = initTime.value;
-    initTime.name = window.HUE_I18n.profile.codegen;
-    initTime.color = colors[2];
-    initTime.start_time = 0;
-    return [initTime].concat(timeline);
+    return [$.extend({}, initTime, { start_time: 0, duration: initTime.value, name: window.HUE_I18n.profile.codegen, color: colors[2] })].concat(timeline);
+  }
+
+  function getHealthData(key, startTime) {
+    var id = getId(key);
+    var node = getNode(id);
+    if (!node || !node.health) {
+      return;
+    }
+    return _timelineAfter(startTime, node.health.map(function (risk) { return $.extend({}, risk, { value: risk.impact }) }));
   }
   }
 
 
   function getFragment(id) {
   function getFragment(id) {
@@ -364,14 +398,21 @@ function impalaDagre(id) {
 
 
   function getScanCPUIOTimelineData(key) {
   function getScanCPUIOTimelineData(key) {
     var cpuExchange = getMaxValue(key, 'LocalTime');
     var cpuExchange = getMaxValue(key, 'LocalTime');
+    cpuExchange.clazz = 'cpu';
     var ioTime = getMaxValue(key, 'ChildTime');
     var ioTime = getMaxValue(key, 'ChildTime');
+    ioTime.clazz = 'io';
     return [ioTime, cpuExchange];
     return [ioTime, cpuExchange];
   }
   }
 
 
   function getTopNodes() {
   function getTopNodes() {
     return Object.keys(states_by_name).map(function (key) {
     return Object.keys(states_by_name).map(function (key) {
       var timeline = getCPUTimelineData(key);
       var timeline = getCPUTimelineData(key);
-      var sumTime = sum(timeline, 'duration')
+      var sumTime;
+      if (timeline) {
+        sumTime = sum(timeline, 'duration');
+      } else {
+        sumTime = states_by_name[key].max_time_val;
+      }
       return { name: states_by_name[key].label, duration: sumTime, unit: 5, key: key, icon: states_by_name[key].icon  };
       return { name: states_by_name[key].label, duration: sumTime, unit: 5, key: key, icon: states_by_name[key].icon  };
     }).sort(function (a, b) {
     }).sort(function (a, b) {
       return b.duration - a.duration;
       return b.duration - a.duration;
@@ -383,6 +424,7 @@ function impalaDagre(id) {
     var node = getNode(id);
     var node = getNode(id);
     var timeline = node.timeline;
     var timeline = node.timeline;
     var cpuExchange = getMaxValue(key, 'LocalTime');
     var cpuExchange = getMaxValue(key, 'LocalTime');
+    cpuExchange = $.extend({}, cpuExchange, { clazz: 'cpu' });
 
 
     var sender = Object.keys(states_by_name).filter(function(node) {
     var sender = Object.keys(states_by_name).filter(function(node) {
       return states_by_name[node].parent == key;
       return states_by_name[node].parent == key;
@@ -391,8 +433,10 @@ function impalaDagre(id) {
       return [{ value: 0, unit: 0 }, { value: 0, unit: 0 }, { value: 0, unit: 0 }];
       return [{ value: 0, unit: 0 }, { value: 0, unit: 0 }, { value: 0, unit: 0 }];
     }
     }
     var networkTime = getMaxTotalNetworkTime(sender, key);
     var networkTime = getMaxTotalNetworkTime(sender, key);
+    networkTime = $.extend({}, networkTime, { clazz: 'io' });
     var krpcTime = getMaxFragmentMetric(sender, 'LocalTime', 'children.KrpcDataStreamSender.hosts');
     var krpcTime = getMaxFragmentMetric(sender, 'LocalTime', 'children.KrpcDataStreamSender.hosts');
-    return [ krpcTime, networkTime, cpuExchange];
+    krpcTime = $.extend({}, krpcTime, { clazz: 'cpu' });
+    return [krpcTime, networkTime, cpuExchange];
   }
   }
 
 
   function getTimelineData(key, name) {
   function getTimelineData(key, name) {
@@ -435,7 +479,8 @@ function impalaDagre(id) {
       var cpuTimelineSection = detailsContent.append('div').classed('details-section', true);
       var cpuTimelineSection = detailsContent.append('div').classed('details-section', true);
       var cpuTimelineTitle = cpuTimelineSection.append('header');
       var cpuTimelineTitle = cpuTimelineSection.append('header');
       cpuTimelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-filter');
       cpuTimelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-filter');
-      cpuTimelineTitle.append('h5').text(window.HUE_I18n.profile.topnodes + ' (' + ko.bindingHandlers.numberFormat.human(getMetricsMax(), 5) + ')');
+      var metricsMax = getMetricsMax() ? ' (' + ko.bindingHandlers.numberFormat.human(getMetricsMax(), 5) + ')' : '';
+      cpuTimelineTitle.append('h5').text(window.HUE_I18n.profile.topnodes + metricsMax);
       var cpuTimelineSectionTable = cpuTimelineSection.append('table').classed('clickable', true);
       var cpuTimelineSectionTable = cpuTimelineSection.append('table').classed('clickable', true);
       var cpuTimelineSectionTableRows = cpuTimelineSectionTable.selectAll('tr').data(topNodes).enter().append('tr').on('click', function (node) {
       var cpuTimelineSectionTableRows = cpuTimelineSectionTable.selectAll('tr').data(topNodes).enter().append('tr').on('click', function (node) {
         select(node.key);
         select(node.key);
@@ -445,26 +490,26 @@ function impalaDagre(id) {
       cpuTimelineSectionTableRows.append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
       cpuTimelineSectionTableRows.append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
     }
     }
 
 
-    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Compilation'), 'Compilation', null);
-    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Timeline'), window.HUE_I18n.profile.execution, null);
+    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Compilation'), window.HUE_I18n.profile.planning, '#hi-access-time');
+    appendTimelineAndLegend(detailsContent, getTimelineData('summary', 'Query Timeline'), window.HUE_I18n.profile.execution, '#hi-access-time');
 
 
     d3.select('.query-plan .details .metric-title').on('click', function () {
     d3.select('.query-plan .details .metric-title').on('click', function () {
       toggleDetails();
       toggleDetails();
     });
     });
   }
   }
 
 
-  function appendTimelineAndLegend(detailsContent, data, title, max) {
+  function appendTimelineAndLegend(detailsContent, data, title, icon, max) {
     var timeline = renderTimeline(data, max);
     var timeline = renderTimeline(data, max);
     if (timeline) {
     if (timeline) {
       var executionSum = sum(data, 'duration');
       var executionSum = sum(data, 'duration');
       var cpuTimelineSection = detailsContent.append('div').classed('details-section', true);
       var cpuTimelineSection = detailsContent.append('div').classed('details-section', true);
       var cpuTimelineTitle = cpuTimelineSection.append('header');
       var cpuTimelineTitle = cpuTimelineSection.append('header');
-      cpuTimelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
+      cpuTimelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', icon);
       cpuTimelineTitle.append('h5').text(title + ' (' + ko.bindingHandlers.numberFormat.human(executionSum, 5) + ')');
       cpuTimelineTitle.append('h5').text(title + ' (' + ko.bindingHandlers.numberFormat.human(executionSum, 5) + ')');
       cpuTimelineSection.node().appendChild($.parseXML(timeline).children[0]);
       cpuTimelineSection.node().appendChild($.parseXML(timeline).children[0]);
 
 
       var cpuTimelineSectionTable = cpuTimelineSection.append('table').classed('column', true);
       var cpuTimelineSectionTable = cpuTimelineSection.append('table').classed('column', true);
-      cpuTimelineSectionTable.append('tr').selectAll('td').data(data).enter().append('td').html(function (time, index) { return '<div class="legend-icon ' + (time.clazz ? time.clazz : '') + '" style="' + (!time.clazz && 'background-color: ' + (time.color || colors[index % colors.length])) + '"></div><div class="metric-name" title="' + time.name + '">' + time.name + '</div>'; });
+      cpuTimelineSectionTable.append('tr').selectAll('td').data(data).enter().append('td').html(function (time, index) { return '<div class="legend-icon ' + (time.clazz ? time.clazz : '') + '" style="' + (!time.clazz && 'background-color: ' + (time.color || colors[index % colors.length])) + '"></div><div class="metric-name" title="' + time.name + (time.message ? ': ' + time.message : '') + '">' + time.name + '</div>'; });
       cpuTimelineSectionTable.append('tr').selectAll('td').data(data).enter().append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
       cpuTimelineSectionTable.append('tr').selectAll('td').data(data).enter().append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
     }
     }
   }
   }
@@ -483,13 +528,18 @@ function impalaDagre(id) {
     details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h4>' + states_by_name[key].label+ '</h4></div>')
     details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h4>' + states_by_name[key].label+ '</h4></div>')
     var detailsContent = details.append('div').classed('details-content', true);
     var detailsContent = details.append('div').classed('details-content', true);
 
 
-    appendTimelineAndLegend(detailsContent, getExecutionTimelineData(key), window.HUE_I18n.profile.execution, getMetricsMax());
-    var timeline = renderTimeline(getTimelineData(key), getMetricsMax());
+    var cpuTimelineData = getCPUTimelineData(key);
+    appendTimelineAndLegend(detailsContent, getExecutionTimelineData(key), window.HUE_I18n.profile.execution, '#hi-microchip', getMetricsMax());
+    appendTimelineAndLegend(detailsContent, getHealthData(key, cpuTimelineData[0] && cpuTimelineData[0].start_time), window.HUE_I18n.profile.risks, '#hi-heart', getMetricsMax());
+
+    var timelineData = getTimelineData(key);
+    var timeline = renderTimeline(timelineData, getMetricsMax());
     if (timeline) {
     if (timeline) {
+      var timelineSum = sum(timelineData, 'duration');
       var timelineSection = detailsContent.append('div').classed('details-section', true);
       var timelineSection = detailsContent.append('div').classed('details-section', true);
       var timelineTitle = timelineSection.append('header');
       var timelineTitle = timelineSection.append('header');
       timelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
       timelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
-      timelineTitle.append('h5').text(window.HUE_I18n.profile.timeline);
+      timelineTitle.append('h5').text(window.HUE_I18n.profile.timeline + ' (' + ko.bindingHandlers.numberFormat.human(timelineSum, 5) + ')');
       timelineSection.node().appendChild($.parseXML(timeline).children[0]);
       timelineSection.node().appendChild($.parseXML(timeline).children[0]);
 
 
       var timelineSectionTable = timelineSection.append('table').classed('column', true);
       var timelineSectionTable = timelineSection.append('table').classed('column', true);

+ 9 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less

@@ -106,6 +106,9 @@
     .name {
     .name {
       text-transform: capitalize;
       text-transform: capitalize;
       font-size: 13px;
       font-size: 13px;
+      width: 115px;
+      overflow: hidden;
+      display: inline-block;
     }
     }
     .detail {
     .detail {
       overflow: hidden;
       overflow: hidden;
@@ -291,6 +294,12 @@
           padding-right: 5px;
           padding-right: 5px;
         }
         }
       }
       }
+      .hi {
+        padding-right: 4px;
+      }
+      .fa {
+        padding-right: 2px;
+      }
       .metrics {
       .metrics {
         max-height: calc(~"100% - 40px");
         max-height: calc(~"100% - 40px");
       }
       }

+ 7 - 6
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -7586,7 +7586,7 @@
       humanTime: function (value) {
       humanTime: function (value) {
         value = value * 1;
         value = value * 1;
         if (value < Math.pow(10, 3)) {
         if (value < Math.pow(10, 3)) {
-          return value + " ns";
+          return sprintf("%i ns", value);
         } else if (value - Math.pow(10, 6) < -Math.pow(10, 3) / 2) { // Make sure rounding doesn't cause numbers to have more than 4 significant digits.
         } else if (value - Math.pow(10, 6) < -Math.pow(10, 3) / 2) { // Make sure rounding doesn't cause numbers to have more than 4 significant digits.
           value = (value * 1.0) / Math.pow(10, 3);
           value = (value * 1.0) / Math.pow(10, 3);
           var sprint = value > 100 ? "%i us" : "%.1f us";
           var sprint = value > 100 ? "%i us" : "%.1f us";
@@ -7604,17 +7604,18 @@
           var buffer = "";
           var buffer = "";
 
 
           if (value > (HOUR)) {
           if (value > (HOUR)) {
-            buffer += sprintf("%i h ", value / HOUR);
+            buffer += sprintf("%i h", value / HOUR);
             value = value % HOUR;
             value = value % HOUR;
           }
           }
 
 
-          if (value > MINUTE) {
-            buffer += sprintf("%i m ", value / MINUTE);
+          if (buffer.length < 4 && value > MINUTE) {
+            var sprint = buffer.length ? " %i m" : "%i m";
+            buffer += sprintf(sprint, value / MINUTE);
             value = value % MINUTE;
             value = value % MINUTE;
           }
           }
 
 
-          if (value > SECOND) {
-            var sprint = buffer.length ? "%i s" : "%.1f s";
+          if (buffer.length < 4 && value > SECOND) {
+            var sprint = buffer.length ? " %i s" : "%.1f s";
             buffer += sprintf(sprint, value * 1.0 / SECOND);
             buffer += sprintf(sprint, value * 1.0 / SECOND);
           }
           }
           return buffer;
           return buffer;

+ 3 - 1
desktop/core/src/desktop/templates/global_js_constants.mako

@@ -185,7 +185,9 @@
       codegen: "${ _('CodeGen') }",
       codegen: "${ _('CodeGen') }",
       overview: "${ _('Overview') }",
       overview: "${ _('Overview') }",
       topnodes: "${ _('Top Nodes') }",
       topnodes: "${ _('Top Nodes') }",
-      compilation: "${ _('Compilation') }"
+      compilation: "${ _('Compilation') }",
+      planning: "${ _('Planning') }",
+      risks: "${ _('Risks') }"
     }
     }
   };
   };
 
 

+ 7 - 0
desktop/core/src/desktop/templates/hue_icons.mako

@@ -384,6 +384,13 @@
     <symbol id="hi-copy" viewBox="0 0 512 512">
     <symbol id="hi-copy" viewBox="0 0 512 512">
       <path d="M433.941 65.941l-51.882-51.882A48 48 0 0 0 348.118 0H176c-26.51 0-48 21.49-48 48v48H48c-26.51 0-48 21.49-48 48v320c0 26.51 21.49 48 48 48h224c26.51 0 48-21.49 48-48v-48h80c26.51 0 48-21.49 48-48V99.882a48 48 0 0 0-14.059-33.941zM266 464H54a6 6 0 0 1-6-6V150a6 6 0 0 1 6-6h74v224c0 26.51 21.49 48 48 48h96v42a6 6 0 0 1-6 6zm128-96H182a6 6 0 0 1-6-6V54a6 6 0 0 1 6-6h106v88c0 13.255 10.745 24 24 24h88v202a6 6 0 0 1-6 6zm6-256h-64V48h9.632c1.591 0 3.117.632 4.243 1.757l48.368 48.368a6 6 0 0 1 1.757 4.243V112z"></path>
       <path d="M433.941 65.941l-51.882-51.882A48 48 0 0 0 348.118 0H176c-26.51 0-48 21.49-48 48v48H48c-26.51 0-48 21.49-48 48v320c0 26.51 21.49 48 48 48h224c26.51 0 48-21.49 48-48v-48h80c26.51 0 48-21.49 48-48V99.882a48 48 0 0 0-14.059-33.941zM266 464H54a6 6 0 0 1-6-6V150a6 6 0 0 1 6-6h74v224c0 26.51 21.49 48 48 48h96v42a6 6 0 0 1-6 6zm128-96H182a6 6 0 0 1-6-6V54a6 6 0 0 1 6-6h106v88c0 13.255 10.745 24 24 24h88v202a6 6 0 0 1-6 6zm6-256h-64V48h9.632c1.591 0 3.117.632 4.243 1.757l48.368 48.368a6 6 0 0 1 1.757 4.243V112z"></path>
     </symbol>
     </symbol>
+    <symbol id="hi-microchip" viewBox="0 0 1024 1024">
+      <path d="M682.67 307.2h-341.34c-18.852 0-34.13 15.281-34.13 34.13v341.34c0 18.852 15.28 34.13 34.13 34.13h341.34c18.852 0 34.13-15.28 34.13-34.13v-341.34c0-18.851-15.28-34.13-34.13-34.13z" fill="" />
+      <path d="M921.606 546.133c18.85 0 34.127-15.282 34.127-34.133 0-18.855-15.279-34.133-34.127-34.133h-68.273v-102.4h68.273c18.85 0 34.127-15.282 34.127-34.133 0-18.855-15.279-34.133-34.127-34.133h-68.273v-68.267c0-37.706-30.564-68.267-68.267-68.267h-68.267v-68.273c0-18.85-15.282-34.127-34.133-34.127-18.855 0-34.133 15.279-34.133 34.127v68.273h-102.4v-68.273c0-18.85-15.282-34.127-34.133-34.127-18.855 0-34.133 15.279-34.133 34.127v68.273h-102.4v-68.273c0-18.85-15.282-34.127-34.133-34.127-18.855 0-34.133 15.279-34.133 34.127v68.273h-68.267c-37.706 0-68.267 30.564-68.267 68.267v68.267h-68.273c-18.85 0-34.127 15.282-34.127 34.133 0 18.855 15.279 34.133 34.127 34.133h68.273v102.4h-68.273c-18.85 0-34.127 15.282-34.127 34.133 0 18.855 15.279 34.133 34.127 34.133h68.273v102.4h-68.273c-18.85 0-34.127 15.282-34.127 34.133 0 18.855 15.279 34.133 34.127 34.133h68.273v68.267c0 37.706 30.564 68.267 68.267 68.267h68.267v68.273c0 18.85 15.282 34.127 34.133 34.127 18.855 0 34.133-15.279 34.133-34.127v-68.273h102.4v68.273c0 18.85 15.282 34.127 34.133 34.127 18.855 0 34.133-15.279 34.133-34.127v-68.273h102.4v68.273c0 18.85 15.282 34.127 34.133 34.127 18.855 0 34.133-15.279 34.133-34.127v-68.273h68.267c37.706 0 68.267-30.564 68.267-68.267v-68.267h68.273c18.85 0 34.127-15.282 34.127-34.133 0-18.855-15.279-34.133-34.127-34.133h-68.273v-102.4h68.273zM785.067 785.067h-511.996c-18.853 0-34.138-15.284-34.138-34.138v-511.996h511.996c18.854 0 34.138 15.284 34.138 34.138v511.996z" fill="" />
+    </symbol>
+    <symbol id="hi-heart" viewBox="0 0 1024 1024">
+      <path d="M886.250667 552.490667 512 927.957333l-374.229333-375.466667C79.786667 505.429333 42.666667 433.536 42.666667 352.896 42.666667 211.029333 157.290667 96.042667 298.666667 96.042667c89.088 0 167.488 45.717333 213.333333 114.986667 45.845333-69.269333 124.245333-114.986667 213.333333-114.986667 141.376 0 256 114.986667 256 256.832C981.333333 433.536 944.213333 505.429333 886.250667 552.490667zM832.341333 458.858667l-138.858667 0-53.888-197.610667-20.586667 5.610667-20.117333-7.104-93.226667 317.034667-101.077333-252.672-20.245333 6.741333-19.989333-7.509333-50.794667 135.509333L192.341333 458.858667l0 42.666667 128 0 19.989333 7.509333 44.672-119.168 107.114667 267.754667 40.469333-13.504-3.477333-8.661333 3.349333 1.173333 85.76-291.562667 44.202667 162.090667 20.586667-5.610667 149.333333 0L832.341333 458.858667z" />
+    </symbol>
   </svg>
   </svg>
 
 
   <script type="text/html" id="app-switcher-icon-template">
   <script type="text/html" id="app-switcher-icon-template">

+ 1 - 0
desktop/libs/libanalyze/reasons/agg_performance.json

@@ -4,6 +4,7 @@
   "metric_names": [
   "metric_names": [
     "LocalTime"
     "LocalTime"
   ],
   ],
+  "unit": 5,
   "rule": {
   "rule": {
     "expr": "vars['LocalTime'] - float(vars['InputRows']) / 0.01",
     "expr": "vars['LocalTime'] - float(vars['InputRows']) / 0.01",
     "message": "Excess time (over expected time) spent in the aggregate; might be caused by complex group by",
     "message": "Excess time (over expected time) spent in the aggregate; might be caused by complex group by",

+ 1 - 1
desktop/libs/libanalyze/reasons/bytes_read_skew.json

@@ -8,7 +8,7 @@
     "max",
     "max",
     "avg"
     "avg"
   ],
   ],
-  "unit_id": 0,
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['IOBound']==True",
     "condition": "vars['IOBound']==True",
     "expr": "(vars['max'] - vars['avg']) / 100000000 / 5",
     "expr": "(vars['max'] - vars['avg']) / 100000000 / 5",

+ 1 - 0
desktop/libs/libanalyze/reasons/join_performance.json

@@ -5,6 +5,7 @@
     "ProbeRows",
     "ProbeRows",
     "ProbeTime"
     "ProbeTime"
   ],
   ],
+  "unit": 5,
   "rule": {
   "rule": {
     "expr": "vars['ProbeTime'] - float(vars['ProbeRows']) / 0.005",
     "expr": "vars['ProbeTime'] - float(vars['ProbeRows']) / 0.005",
     "message": "Excess time (over expected time) spent in the hash join",
     "message": "Excess time (over expected time) spent in the hash join",

+ 1 - 1
desktop/libs/libanalyze/reasons/metadata_missing.json

@@ -5,7 +5,7 @@
     "MetadataLoadTime",
     "MetadataLoadTime",
     "PlanningTime"
     "PlanningTime"
   ],
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "float(vars['MetadataLoadTime']) / float(vars['PlanningTime']) > 0.5 and vars['MetadataLoadTime'] > 2000000000",
     "condition": "float(vars['MetadataLoadTime']) / float(vars['PlanningTime']) > 0.5 and vars['MetadataLoadTime'] > 2000000000",
     "expr": "vars['MetadataLoadTime']",
     "expr": "vars['MetadataLoadTime']",

+ 1 - 1
desktop/libs/libanalyze/reasons/remote_scan_ranges.json

@@ -4,7 +4,7 @@
   "metric_names": [
   "metric_names": [
     "BytesReadRemoteUnexpected"
     "BytesReadRemoteUnexpected"
   ],
   ],
-  "unit_id": 0,
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['IOBound']==True",
     "condition": "vars['IOBound']==True",
     "expr": "vars['BytesReadRemoteUnexpected'] * (1/30 - 1/100) * 1/1024/1024",
     "expr": "vars['BytesReadRemoteUnexpected'] * (1/30 - 1/100) * 1/1024/1024",

+ 1 - 1
desktop/libs/libanalyze/reasons/rows_read_skew.json

@@ -8,7 +8,7 @@
     "max",
     "max",
     "avg"
     "avg"
   ],
   ],
-  "unit_id": 0,
+  "unit": 5,
   "rule": {
   "rule": {
     "expr": "(vars['max'] - vars['avg']) / 40.0 * 1000",
     "expr": "(vars['max'] - vars['avg']) / 40.0 * 1000",
     "message": "The skew (max-avg) in rows processed",
     "message": "The skew (max-avg) in rows processed",

+ 1 - 1
desktop/libs/libanalyze/reasons/scan_performance.json

@@ -7,7 +7,7 @@
     "ScannerThreadsSysTime",
     "ScannerThreadsSysTime",
     "AverageScannerThreadConcurrency"
     "AverageScannerThreadConcurrency"
   ],
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
   "rule": {
     "expr": "(vars['ScannerThreadsUserTime'] + vars['ScannerThreadsSysTime'] - vars['RowsRead'] * 100) / max(1,vars['AverageScannerThreadConcurrency'])",
     "expr": "(vars['ScannerThreadsUserTime'] + vars['ScannerThreadsSysTime'] - vars['RowsRead'] * 100) / max(1,vars['AverageScannerThreadConcurrency'])",
     "message": "Predicates might be expensive (expectes speed 10m rows per sec per core)",
     "message": "Predicates might be expensive (expectes speed 10m rows per sec per core)",

+ 2 - 1
desktop/libs/libanalyze/reasons/scanner_filter.json

@@ -4,9 +4,10 @@
   "metric_names": [
   "metric_names": [
     "RowsRead", "RowsReturned"
     "RowsRead", "RowsReturned"
   ],
   ],
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['RowsReturned'] < vars['RowsRead']",
     "condition": "vars['RowsReturned'] < vars['RowsRead']",
-    "expr": "vars['RowsRead'] - vars['RowsReturned']",
+    "expr": "(vars['RowsRead'] - vars['RowsReturned']) / 40.0 * 1000",
     "message": "Kudu could not evaluate the predicate and it was evaluated by Impala. Kudu supports predicates =, <=, <, >, >=, BETWEEN, or IN.",
     "message": "Kudu could not evaluate the predicate and it was evaluated by Impala. Kudu supports predicates =, <=, <, >, >=, BETWEEN, or IN.",
     "label": "Kudu predicate optimization"
     "label": "Kudu predicate optimization"
   },
   },

+ 1 - 1
desktop/libs/libanalyze/reasons/scanner_parallelism.json

@@ -5,7 +5,7 @@
     "AverageScannerThreadConcurrency",
     "AverageScannerThreadConcurrency",
     "LocalTime"
     "LocalTime"
   ],
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['IOBound']==False",
     "condition": "vars['IOBound']==False",
     "expr": "float(8 - vars['AverageScannerThreadConcurrency']) / 8.0 * vars['LocalTime']",
     "expr": "float(8 - vars['AverageScannerThreadConcurrency']) / 8.0 * vars['LocalTime']",

+ 1 - 0
desktop/libs/libanalyze/reasons/selective_scan.json

@@ -6,6 +6,7 @@
     "RowsReturned",
     "RowsReturned",
     "LocalTime"
     "LocalTime"
   ],
   ],
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['RowsRead']>8000000",
     "condition": "vars['RowsRead']>8000000",
     "expr": "float(vars['RowsRead'] - vars['RowsReturned']) / vars['LocalTime']",
     "expr": "float(vars['RowsRead'] - vars['RowsReturned']) / vars['LocalTime']",

+ 1 - 1
desktop/libs/libanalyze/reasons/skew.json

@@ -6,7 +6,7 @@
     "max",
     "max",
     "avg"
     "avg"
   ],
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
   "rule": {
     "message": "The skew (max-avg) contributed this amount of time to this SQL operator",
     "message": "The skew (max-avg) contributed this amount of time to this SQL operator",
     "expr": "(vars['max'] - vars['avg'])",
     "expr": "(vars['max'] - vars['avg'])",

+ 1 - 1
desktop/libs/libanalyze/reasons/slow_table_sink.json

@@ -6,7 +6,7 @@
     "BytesWritten",
     "BytesWritten",
     "LocalTime"
     "LocalTime"
   ],
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['BytesWritten'] > 0",
     "condition": "vars['BytesWritten'] > 0",
     "expr": "float(vars['LocalTime']) - 0.01 / float(vars['BytesWritten'])",
     "expr": "float(vars['LocalTime']) - 0.01 / float(vars['BytesWritten'])",

+ 1 - 1
desktop/libs/libanalyze/reasons/sort_performance.json

@@ -4,7 +4,7 @@
   "metric_names": [
   "metric_names": [
     "LocalTime"
     "LocalTime"
   ],
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
   "rule": {
     "expr": "vars['LocalTime'] - float(vars['InputRows']) / 0.01",
     "expr": "vars['LocalTime'] - float(vars['InputRows']) / 0.01",
     "message": "Excess time (over expected time) spent in the sort; might be caused by too many sorting column",
     "message": "Excess time (over expected time) spent in the sort; might be caused by too many sorting column",

+ 6 - 5
desktop/libs/libanalyze/reasons/spilling.json

@@ -1,11 +1,12 @@
 {
 {
   "type": "SQLOperator",
   "type": "SQLOperator",
-  "node_name": ["HashJoinNode", "AGGREGATION_NODE"],
-  "metric_names": "SpilledPartitions",
+  "node_name": ["HASH_JOIN_NODE", "AGGREGATION_NODE"],
+  "metric_names": "SpillTime",
+  "unit": 5,
   "rule": {
   "rule": {
-    "condition": "vars['SpilledPartitions'] > 0",
-    "expr": "1",
-    "message": "This operation has spilled to disk. Check if the ressource configuration of Impala can be changed to allow for a higher memory limit.",
+    "condition": "vars['SpillTime'] > 0",
+    "expr": "vars['SpillTime']",
+    "message": "This operation has spilled to disk. Check if the resource configuration of Impala can be changed to allow for a higher memory limit.",
     "label": " Spilled Partitions"
     "label": " Spilled Partitions"
   },
   },
   "fix": {
   "fix": {

+ 3 - 2
desktop/libs/libanalyze/reasons/stats_missing.json

@@ -2,12 +2,13 @@
   "type": "SQLOperator",
   "type": "SQLOperator",
   "node_name": ["HDFS_SCAN_NODE", "KUDU_SCAN_NODE", "HBASE_SCAN_NODE"],
   "node_name": ["HDFS_SCAN_NODE", "KUDU_SCAN_NODE", "HBASE_SCAN_NODE"],
   "metric_names": [
   "metric_names": [
-    "MissingStats"
+    "MissingStats", "TotalTime"
   ],
   ],
   "info_names": ["Table"],
   "info_names": ["Table"],
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['MissingStats'] == 1",
     "condition": "vars['MissingStats'] == 1",
-    "expr": "1",
+    "expr": "vars['TotalTime'] * 0.2",
     "message": "The statistics are missing or corrupt which prevent scan optimizations.",
     "message": "The statistics are missing or corrupt which prevent scan optimizations.",
     "label": "Statistics Missing"
     "label": "Statistics Missing"
   },
   },

+ 1 - 1
desktop/libs/libanalyze/reasons/too_many_columns.json

@@ -6,7 +6,7 @@
     "NumColumns",
     "NumColumns",
     "LocalTime"
     "LocalTime"
   ],
   ],
-  "unit_id": 5,
+  "unit": 5,
   "rule": {
   "rule": {
     "condition": "vars['NumColumns'] > 15",
     "condition": "vars['NumColumns'] > 15",
     "expr": "float(vars['NumColumns'] - 15) / float(vars['NumColumns']) * vars['LocalTime']",
     "expr": "float(vars['NumColumns'] - 15) / float(vars['NumColumns']) * vars['LocalTime']",

+ 6 - 0
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -317,15 +317,18 @@ def metrics(profile):
   def flatten(node, counter_map=counter_map):
   def flatten(node, counter_map=counter_map):
     is_plan_node = node.is_plan_node()
     is_plan_node = node.is_plan_node()
     is_parent_node = is_plan_node
     is_parent_node = is_plan_node
+    bIncludeInMinMax = True
     if not is_plan_node:
     if not is_plan_node:
       if node.plan_node:
       if node.plan_node:
         nid = node.plan_node.id()
         nid = node.plan_node.id()
       elif node.is_fragment_instance():
       elif node.is_fragment_instance():
         is_parent_node = True
         is_parent_node = True
         nid = node.fragment.id()
         nid = node.fragment.id()
+        bIncludeInMinMax = False
       elif node.is_fragment() and node.is_averaged():
       elif node.is_fragment() and node.is_averaged():
         is_parent_node = True
         is_parent_node = True
         nid = node.id()
         nid = node.id()
+        bIncludeInMinMax = False
       elif node.fragment:
       elif node.fragment:
         nid = node.fragment.id()
         nid = node.fragment.id()
       else:
       else:
@@ -348,6 +351,7 @@ def metrics(profile):
         counter_map['nodes'][nid]['other'] = plan_json[nid]
         counter_map['nodes'][nid]['other'] = plan_json[nid]
       if is_plan_node:
       if is_plan_node:
         counter_map['nodes'][nid]['fragment'] = node.fragment.id()
         counter_map['nodes'][nid]['fragment'] = node.fragment.id()
+      counter_map['nodes'][nid]['timeline']['minmax'] = bIncludeInMinMax
     else:
     else:
       name = node.name()
       name = node.name()
       if counter_map['nodes'][nid]['children'].get(name) is None:
       if counter_map['nodes'][nid]['children'].get(name) is None:
@@ -359,6 +363,8 @@ def metrics(profile):
   for nodeid, node in counter_map['nodes'].iteritems():
   for nodeid, node in counter_map['nodes'].iteritems():
     host_min = {'value': sys.maxint, 'host' : None}
     host_min = {'value': sys.maxint, 'host' : None}
     host_max = {'value': -(sys.maxint - 1), 'host' : None}
     host_max = {'value': -(sys.maxint - 1), 'host' : None}
+    if not node['timeline']['minmax']:
+      continue
     for host_name, host_value in node['timeline']['hosts'].iteritems():
     for host_name, host_value in node['timeline']['hosts'].iteritems():
       for event_name, event in host_value.iteritems():
       for event_name, event in host_value.iteritems():
         if len(event):
         if len(event):

+ 22 - 4
desktop/libs/libanalyze/src/libanalyze/models.py

@@ -33,6 +33,7 @@ class Contributor(object):
 
 
 class Reason(object):
 class Reason(object):
   def __init__(self, **kwargs):
   def __init__(self, **kwargs):
+    self.name = None
     self.message = None
     self.message = None
     self.impact = None
     self.impact = None
     self.unit = None
     self.unit = None
@@ -56,11 +57,28 @@ def query_node_by_id(profile, node_id, metric_name, averaged=False):
   result = profile.find_by_id(node_id)
   result = profile.find_by_id(node_id)
   if not result:
   if not result:
     return result
     return result
-  nodes = filter(lambda x: x.fragment.is_averaged() == averaged, result)
+
+  nodes = _filter_averaged(result, averaged)
   metric = reduce(lambda x, y: x + y.find_metric_by_name(metric_name), nodes, [])
   metric = reduce(lambda x, y: x + y.find_metric_by_name(metric_name), nodes, [])
 
 
   return map(lambda x: L(x['value'], x['unit'], 0, x['node'].fragment.id(), x['node'].host(), 0, x['node'].id(), x['node'].name(), value=x['value'], unit=x['unit'], fragment_id=0, fid=x['node'].fragment.id(), host=x['node'].host(), node_id=x['node'].id(), name=x['node'].name(), node=x['node']), metric)
   return map(lambda x: L(x['value'], x['unit'], 0, x['node'].fragment.id(), x['node'].host(), 0, x['node'].id(), x['node'].name(), value=x['value'], unit=x['unit'], fragment_id=0, fid=x['node'].fragment.id(), host=x['node'].host(), node_id=x['node'].id(), name=x['node'].name(), node=x['node']), metric)
 
 
+def _filter_averaged(result, averaged=False):
+  #nodes = filter(lambda x: x.fragment.is_averaged() == averaged, result)
+  # Averaged results are not always present. If we're looking for averaged results, sort by averaged and get first result (hopefully getting averaged!).
+  # If we're not looking for averaged results, remove them.
+  if averaged:
+    def by_averaged(x, y):
+      if x.fragment.is_averaged():
+        return -1
+      elif y.fragment.is_averaged():
+        return 1
+      else:
+        return 0
+    return sorted(result, cmp=by_averaged)
+  else:
+    return filter(lambda x: x.fragment.is_averaged() == averaged, result)
+
 def query_node_by_metric(profile, node_name, metric_name):
 def query_node_by_metric(profile, node_name, metric_name):
   """Given the query_id, searches for the corresponding query profile and
   """Given the query_id, searches for the corresponding query profile and
   selects the node instances given by node_name, selects the metric given by
   selects the node instances given by node_name, selects the metric given by
@@ -91,7 +109,7 @@ def query_element_by_info(profile, node_name, metric_name):
   metric = reduce(lambda x, y: x + y.find_info_by_name(metric_name), nodes, [])
   metric = reduce(lambda x, y: x + y.find_info_by_name(metric_name), nodes, [])
   return map(lambda x: L(x['value'], 0, x['node'].fragment.id() if x['node'].fragment else '', x['node'].host(), 0, x['node'].id(), x['node'].name(), value=x['value'], fragment_id=0, fid=x['node'].fragment.id() if x['node'].fragment else '', host=x['node'].host(), node_id=x['node'].id(), name=x['node'].name(), node=x['node']), metric)
   return map(lambda x: L(x['value'], 0, x['node'].fragment.id() if x['node'].fragment else '', x['node'].host(), 0, x['node'].id(), x['node'].name(), value=x['value'], fragment_id=0, fid=x['node'].fragment.id() if x['node'].fragment else '', host=x['node'].host(), node_id=x['node'].id(), name=x['node'].name(), node=x['node']), metric)
 
 
-def query_avg_fragment_metric_by_node_nid(profile, node_nid, metric_name):
+def query_avg_fragment_metric_by_node_nid(profile, node_nid, metric_name, default):
   """
   """
   Given the surragate node id (i.e. unique id of the plan node in the database),
   Given the surragate node id (i.e. unique id of the plan node in the database),
   return the value of the fragment level metric.
   return the value of the fragment level metric.
@@ -102,9 +120,9 @@ def query_avg_fragment_metric_by_node_nid(profile, node_nid, metric_name):
   result = profile.find_by_id(node_nid)
   result = profile.find_by_id(node_nid)
   if not result:
   if not result:
     return result
     return result
-  node = map(lambda x: x, filter(lambda x: x.fragment.is_averaged() == True, result))[0]
+  node = _filter_averaged(result, True)[0]
   metric = node.fragment.find_metric_by_name(metric_name)
   metric = node.fragment.find_metric_by_name(metric_name)
-  return metric[0]['value']
+  return metric and metric[0]['value'] or default
 
 
 def query_fragment_metric_by_node_id(node, metric_name):
 def query_fragment_metric_by_node_id(node, metric_name):
   """
   """

+ 34 - 14
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -170,10 +170,10 @@ class SQLOperatorReason:
                 local_vars['vars'].update(dict(zip(self.kwargs['info_names'], metric_values)))
                 local_vars['vars'].update(dict(zip(self.kwargs['info_names'], metric_values)))
                 expr_data = exprs.Expr.evaluate(self.kwargs['fix']['data'], local_vars)
                 expr_data = exprs.Expr.evaluate(self.kwargs['fix']['data'], local_vars)
 
 
-        msg = self.rule["label"] + ": " + self.rule["message"]
         return {
         return {
             "impact": impact,
             "impact": impact,
-            "message": msg,
+            "message": self.rule["message"],
+            "label": self.rule["label"],
             "data": expr_data
             "data": expr_data
         }
         }
 
 
@@ -238,10 +238,10 @@ class SummaryReason(SQLOperatorReason):
                     if (impact is None or impact < expr_val):
                     if (impact is None or impact < expr_val):
                         impact = expr_val
                         impact = expr_val
 
 
-        msg = self.rule["label"] + ": " + self.rule["message"]
         return {
         return {
             "impact": impact,
             "impact": impact,
-            "message": msg
+            "message": self.rule["message"],
+            "label": self.rule["label"]
         }
         }
 
 
 class JoinOrderStrategyCheck(SQLOperatorReason):
 class JoinOrderStrategyCheck(SQLOperatorReason):
@@ -282,7 +282,8 @@ class JoinOrderStrategyCheck(SQLOperatorReason):
         if (impact > 0):
         if (impact > 0):
             return {
             return {
                 "impact": impact,
                 "impact": impact,
-                "message": "Wrong join order - RHS %d; LHS %d" % (rhsRows, lhsRows)
+                "message": "RHS %d; LHS %d" % (rhsRows, lhsRows),
+                "label": "Wrong join order"
             }
             }
 
 
         bcost = rhsRows * hosts
         bcost = rhsRows * hosts
@@ -290,7 +291,8 @@ class JoinOrderStrategyCheck(SQLOperatorReason):
         impact = (networkcost - min(bcost, scost) - 1) / hosts / 0.01
         impact = (networkcost - min(bcost, scost) - 1) / hosts / 0.01
         return {
         return {
             "impact": impact,
             "impact": impact,
-            "message": "Wrong join strategy - RHS %d; LHS %d" % (rhsRows, lhsRows)
+            "message": "RHS %d; LHS %d" % (rhsRows, lhsRows),
+            "label": "Wrong join strategy"
         }
         }
 
 
 class ExplodingJoinCheck(SQLOperatorReason):
 class ExplodingJoinCheck(SQLOperatorReason):
@@ -319,7 +321,8 @@ class ExplodingJoinCheck(SQLOperatorReason):
             impact = probeTime * (rowsReturned - probeRows) / rowsReturned
             impact = probeTime * (rowsReturned - probeRows) / rowsReturned
         return {
         return {
             "impact": impact,
             "impact": impact,
-            "message": "Exploding join: %d input rows are exploded to %d output rows" % (probeRows, rowsReturned)
+            "message": "%d input rows are exploded to %d output rows" % (probeRows, rowsReturned),
+            "label": "Exploding join"
         }
         }
 
 
 class NNRpcCheck(SQLOperatorReason):
 class NNRpcCheck(SQLOperatorReason):
@@ -336,14 +339,15 @@ class NNRpcCheck(SQLOperatorReason):
         }
         }
         :return:
         :return:
         """
         """
-        totalStorageTime = models.query_avg_fragment_metric_by_node_nid(profile, plan_node_id, "TotalStorageWaitTime")
+        totalStorageTime = models.query_avg_fragment_metric_by_node_nid(profile, plan_node_id, "TotalStorageWaitTime", 0)
         hdfsRawReadTime = models.query_node_by_id(profile, plan_node_id, "TotalRawHdfsReadTime(*)", True)[0][0]
         hdfsRawReadTime = models.query_node_by_id(profile, plan_node_id, "TotalRawHdfsReadTime(*)", True)[0][0]
         avgReadThreads = models.query_node_by_id(profile, plan_node_id, "AverageHdfsReadThreadConcurrency", True)[0][0]
         avgReadThreads = models.query_node_by_id(profile, plan_node_id, "AverageHdfsReadThreadConcurrency", True)[0][0]
         avgReadThreads = max(1, to_double(avgReadThreads))
         avgReadThreads = max(1, to_double(avgReadThreads))
         impact = max(0, (totalStorageTime - hdfsRawReadTime) / avgReadThreads)
         impact = max(0, (totalStorageTime - hdfsRawReadTime) / avgReadThreads)
         return {
         return {
             "impact": impact,
             "impact": impact,
-            "message": "This is the time waiting for HDFS NN RPC."
+            "message": "This is the time waiting for HDFS NN RPC.",
+            "label": "HDFS NN RPC"
         }
         }
 
 
 class TopDownAnalysis:
 class TopDownAnalysis:
@@ -488,7 +492,7 @@ class TopDownAnalysis:
                 fix.update(cause.kwargs['fix'])
                 fix.update(cause.kwargs['fix'])
                 if evaluation.get('data'):
                 if evaluation.get('data'):
                   fix['data'] = evaluation['data']
                   fix['data'] = evaluation['data']
-                reason = models.Reason(message=evaluation['message'], impact=evaluation['impact'], unit=cause.kwargs.get('unit_id', ''), fix=fix)
+                reason = models.Reason(name=evaluation['label'], message=evaluation['message'], impact=evaluation['impact'], unit=cause.kwargs.get('unit', ''), fix=fix)
                 reasons.append(reason)
                 reasons.append(reason)
         return sorted(reasons, key=lambda x: x.impact, reverse=True)
         return sorted(reasons, key=lambda x: x.impact, reverse=True)
 
 
@@ -614,7 +618,8 @@ class TopDownAnalysis:
             counter_map = node.counter_map()
             counter_map = node.counter_map()
 
 
             # Load the metric data as if the object would be loaded from the DB
             # Load the metric data as if the object would be loaded from the DB
-            local_time = counter_map['TotalTime'].value - child_time
+            local_time = max(counter_map['TotalTime'].value - child_time, 0)
+            has_spilled = False
 
 
             # Make sure to substract the wait time for the exchange node
             # Make sure to substract the wait time for the exchange node
             if is_plan_node and re.search(r'EXCHANGE_NODE', node.val.name) is not None:
             if is_plan_node and re.search(r'EXCHANGE_NODE', node.val.name) is not None:
@@ -634,14 +639,24 @@ class TopDownAnalysis:
             if re.search(r'KUDU_SCAN_NODE', node.val.name):
             if re.search(r'KUDU_SCAN_NODE', node.val.name):
               child_time = counter_map.get('KuduClientTime', models.TCounter(value=0)).value
               child_time = counter_map.get('KuduClientTime', models.TCounter(value=0)).value
               local_time = counter_map['TotalTime'].value
               local_time = counter_map['TotalTime'].value
-              counter_map['TotalTime'].value = child_time + local_time
             if re.search(r'HDFS_SCAN_NODE', node.val.name):
             if re.search(r'HDFS_SCAN_NODE', node.val.name):
               child_time = counter_map.get('TotalRawHdfsReadTime(*)', models.TCounter(value=0)).value
               child_time = counter_map.get('TotalRawHdfsReadTime(*)', models.TCounter(value=0)).value
               local_time = counter_map['TotalTime'].value
               local_time = counter_map['TotalTime'].value
-              counter_map['TotalTime'].value = local_time + child_time
+            if re.search(r'Buffer pool', node.val.name):
+              local_time = counter_map.get('WriteIoWaitTime', models.TCounter(value=0)).value + counter_map.get('ReadIoWaitTime', models.TCounter(value=0)).value + counter_map.get('AllocTime', models.TCounter(value=0)).value
+            if counter_map.get('SpilledPartitions', 0) > 0:
+              has_spilled = True
+
+            if re.search(r'AGGREGATION', node.val.name):
+              grouping_aggregator = node.find_by_name('GroupingAggregator')
+              if grouping_aggregator and grouping_aggregator.counter_map().get('SpilledPartitions', models.TCounter(value=0)).value > 0:
+                has_spilled = True
 
 
             # For Hash Join, if the "LocalTime" metrics
             # For Hash Join, if the "LocalTime" metrics
             if is_plan_node and re.search(r'HASH_JOIN_NODE', node.val.name) is not None:
             if is_plan_node and re.search(r'HASH_JOIN_NODE', node.val.name) is not None:
+                hash_join_builder = node.find_by_name('Hash Join Builder')
+                if hash_join_builder and hash_join_builder.counter_map().get('SpilledPartitions', models.TCounter(value=0)).value > 0:
+                  has_spilled = True
                 if ("LocalTime" in counter_map):
                 if ("LocalTime" in counter_map):
                     local_time = counter_map["LocalTime"].value
                     local_time = counter_map["LocalTime"].value
                 else:
                 else:
@@ -649,13 +664,18 @@ class TopDownAnalysis:
                         counter_map["BuildTime"].value
                         counter_map["BuildTime"].value
 
 
             # Add two virtual metrics for local_time and child_time
             # Add two virtual metrics for local_time and child_time
+            if has_spilled:
+              spill_time = 0
+              buffer_pool = node.find_by_name('Buffer pool')
+              if buffer_pool:
+                spill_time = buffer_pool.counter_map()['LocalTime'].value
+              node.val.counters.append(models.TCounter(name='SpillTime', value=spill_time, unit=5))
             node.val.counters.append(models.TCounter(name='LocalTime', value=local_time, unit=5))
             node.val.counters.append(models.TCounter(name='LocalTime', value=local_time, unit=5))
             node.val.counters.append(models.TCounter(name='ChildTime', value=child_time, unit=5))
             node.val.counters.append(models.TCounter(name='ChildTime', value=child_time, unit=5))
 
 
         profile.foreach_lambda(add_host)
         profile.foreach_lambda(add_host)
 
 
     def run(self, profile):
     def run(self, profile):
-        self.pre_process(profile)
         contributors = self.process(profile)
         contributors = self.process(profile)
         topContributors = self.getTopContributor(100, contributors)
         topContributors = self.getTopContributor(100, contributors)
 
 

Some files were not shown because too many files changed in this diff