Browse Source

Merge remote-tracking branch 'cauldron/upstream-master' into cauldron-cdh6.x

Change-Id: I9f8f1204542bd8db3e8d45f2942a0d0d10293260
Romain Rigaux 6 years ago
parent
commit
667f1eb20e

+ 7 - 2
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -178,7 +178,7 @@ class QueryApi(Api):
     query = self.api.get_query(query_id=appid)
     query = self.api.get_query(query_id=appid)
     query['summary'] = query.get('summary').strip() if query.get('summary') else ''
     query['summary'] = query.get('summary').strip() if query.get('summary') else ''
     query['plan'] = query.get('plan').strip() if query.get('plan') else ''
     query['plan'] = query.get('plan').strip() if query.get('plan') else ''
-    if query['plan_json']:
+    if query.get('plan_json'):
       def get_exchange_icon (o):
       def get_exchange_icon (o):
         if re.search(r'broadcast', o['label_detail'], re.IGNORECASE):
         if re.search(r'broadcast', o['label_detail'], re.IGNORECASE):
           return { 'svg': 'hi-broadcast' }
           return { 'svg': 'hi-broadcast' }
@@ -186,6 +186,11 @@ class QueryApi(Api):
           return { 'font': 'fa-random' }
           return { 'font': 'fa-random' }
         else:
         else:
           return { 'font': 'fa-exchange' }
           return { 'font': 'fa-exchange' }
+      def get_sigma_icon (o):
+        if re.search(r'streaming', o['label_detail'], re.IGNORECASE):
+          return { 'svg': 'hi-sigma-stream' }
+        else:
+          return { 'svg': 'hi-sigma' }
       mapping = {
       mapping = {
         'TOP-N': { 'type': 'TOPN', 'icon': { 'svg': 'hi-filter' } },
         'TOP-N': { 'type': 'TOPN', 'icon': { 'svg': 'hi-filter' } },
         'SORT': { 'type': 'SORT', 'icon': { 'svg': 'hi-sort' } },
         'SORT': { 'type': 'SORT', 'icon': { 'svg': 'hi-sort' } },
@@ -195,7 +200,7 @@ class QueryApi(Api):
         'SCAN KUDU': { 'type': 'SCAN_KUDU', 'icon': { 'font': 'fa-table' } },
         'SCAN KUDU': { 'type': 'SCAN_KUDU', 'icon': { 'font': 'fa-table' } },
         'SCAN HBASE': { 'type': 'SCAN_HBASE', 'icon': { 'font': 'fa-th-large' } },
         'SCAN HBASE': { 'type': 'SCAN_HBASE', 'icon': { 'font': 'fa-th-large' } },
         'HASH JOIN': { 'type': 'HASH_JOIN', 'icon': { 'svg': 'hi-join' } },
         'HASH JOIN': { 'type': 'HASH_JOIN', 'icon': { 'svg': 'hi-join' } },
-        'AGGREGATE': { 'type': 'AGGREGATE', 'icon': { 'svg': 'hi-sigma' } },
+        'AGGREGATE': { 'type': 'AGGREGATE', 'icon': { 'fn': get_sigma_icon } },
         'NESTED LOOP JOIN': { 'type': 'LOOP_JOIN', 'icon': { 'svg': 'hi-nested-loop' } },
         'NESTED LOOP JOIN': { 'type': 'LOOP_JOIN', 'icon': { 'svg': 'hi-nested-loop' } },
         'SUBPLAN': { 'type': 'SUBPLAN', 'icon': { 'svg': 'hi-map' } },
         'SUBPLAN': { 'type': 'SUBPLAN', 'icon': { 'svg': 'hi-map' } },
         'UNNEST': { 'type': 'UNNEST', 'icon': { 'svg': 'hi-unnest' } },
         'UNNEST': { 'type': 'UNNEST', 'icon': { 'svg': 'hi-unnest' } },

File diff suppressed because it is too large
+ 0 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css


+ 139 - 77
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -24,7 +24,6 @@ function impalaDagre(id) {
     },
     },
     metrics: function(data) {
     metrics: function(data) {
       _impalaDagree._metrics = data;
       _impalaDagree._metrics = data;
-      renderGraph()
     },
     },
     update: function(plan) {
     update: function(plan) {
       _impalaDagree._plan = plan;
       _impalaDagree._plan = plan;
@@ -110,38 +109,49 @@ function impalaDagre(id) {
                       "#991F00", "#B22400", "#CC2900", "#E62E00", "#FF3300", "#FF4719"];
                       "#991F00", "#B22400", "#CC2900", "#E62E00", "#FF3300", "#FF4719"];
 
 
   // Recursively build a list of edges and states that comprise the plan graph
   // Recursively build a list of edges and states that comprise the plan graph
-  function build(node, parent, edges, states, colour_idx, max_node_time) {
+  function build(node, parent, edges, states, colour_idx, max_node_time, index, count) {
     if (node["output_card"] === null || node["output_card"] === undefined) {
     if (node["output_card"] === null || node["output_card"] === undefined) {
       return;
       return;
     }
     }
+    var id = getId(node["label"]);
+    var metric_node = _impalaDagree._metrics && _impalaDagree._metrics.nodes[id]
+    var predicates = metric_node && (metric_node.other['group by'] || metric_node.other['hash predicates'] || metric_node.other['predicates']) || '';
     states.push({ "name": node["label"],
     states.push({ "name": node["label"],
                   "type": node["type"],
                   "type": node["type"],
                   "label": node["name"],
                   "label": node["name"],
                   "detail": node["label_detail"],
                   "detail": node["label_detail"],
+                  "predicates": predicates,
                   "num_instances": node["num_instances"],
                   "num_instances": node["num_instances"],
                   "num_active": node["num_active"],
                   "num_active": node["num_active"],
-                  "max_time": node["max_time"],
+                  "max_time": ko.bindingHandlers.numberFormat.human(node["max_time_val"], 5),
                   "avg_time": node["avg_time"],
                   "avg_time": node["avg_time"],
                   "icon": node["icon"],
                   "icon": node["icon"],
                   "is_broadcast": node["is_broadcast"],
                   "is_broadcast": node["is_broadcast"],
-                  "max_time_val": node["max_time_val"]});
+                  "max_time_val": node["max_time_val"],
+                  "width": "200px"});
+    var edgeCount;
     if (parent) {
     if (parent) {
-      var label_val = "" + ko.bindingHandlers.simplesize.humanSize(parseInt(node["output_card"], 10));
-      edges.push({ start: node["label"], end: parent,
-                   style: { label: label_val }});
+      edgeCount = parseInt(node["output_card"], 10);
+      var label_val = "" + ko.bindingHandlers.simplesize.humanSize(edgeCount);
+      edges.push({ start: node["label"], end: parent, style: { label: label_val, labelpos: index === 0 && count > 1 ? 'l' : 'r' }, content: { value: edgeCount, unit: 0 } });
     }
     }
     // Add an inter-fragment edge. We use a red dashed line to show that rows are crossing
     // Add an inter-fragment edge. We use a red dashed line to show that rows are crossing
     // the fragment boundary.
     // the fragment boundary.
     if (node["data_stream_target"]) {
     if (node["data_stream_target"]) {
+      edgeCount = parseInt(node["output_card"], 10);
+      var sendTime = getMaxTotalNetworkSendTime(node["label"]);
+      var text = sendTime && ko.bindingHandlers.numberFormat.human(sendTime.value, sendTime.unit) || ko.bindingHandlers.simplesize.humanSize(edgeCount);
       edges.push({ "start": node["label"],
       edges.push({ "start": node["label"],
                    "end": node["data_stream_target"],
                    "end": node["data_stream_target"],
-                   "style": { label: ko.bindingHandlers.simplesize.humanSize(parseInt(node["output_card"], 10)),
-                              style: "stroke-dasharray: 5, 5;"}});
+                   "content": sendTime ? sendTime : { value: edgeCount, unit: 0 },
+                   "style": { label: text,
+                              style: "stroke-dasharray: 5, 5;",
+                              labelpos: index === 0 && count > 1 ? 'l' : 'r' }});
     }
     }
     max_node_time = Math.max(node["max_time_val"], max_node_time)
     max_node_time = Math.max(node["max_time_val"], max_node_time)
     for (var i = 0; i < node["children"].length; ++i) {
     for (var i = 0; i < node["children"].length; ++i) {
       max_node_time = build(
       max_node_time = build(
-        node["children"][i], node["label"], edges, states, colour_idx, max_node_time);
+        node["children"][i], node["label"], edges, states, colour_idx, max_node_time, i, node["children"].length);
     }
     }
     return max_node_time;
     return max_node_time;
   }
   }
@@ -162,13 +172,17 @@ function impalaDagre(id) {
     $("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
     $("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
   }
   }
 
 
+  function getId(key) {
+    return parseInt(key.split(':')[0], 10);
+  }
+
   function getKey(node) {
   function getKey(node) {
     var nodes = g.nodes();
     var nodes = g.nodes();
     var key;
     var key;
     var nNode = parseInt(node, 10);
     var nNode = parseInt(node, 10);
     var keys = Object.keys(nodes);
     var keys = Object.keys(nodes);
     for (var i = 0; i < keys.length; i++) {
     for (var i = 0; i < keys.length; i++) {
-      if (parseInt(nodes[keys[i]].split(':')[0], 10) == nNode) {
+      if (getId(nodes[keys[i]]) == nNode) {
         key = nodes[keys[i]];
         key = nodes[keys[i]];
         break;
         break;
       }
       }
@@ -203,69 +217,69 @@ function impalaDagre(id) {
     return html;
     return html;
   }
   }
 
 
+  function getMaxTotalNetworkSendTime(node) {
+    var id = getId(node);
+    if (!_impalaDagree._metrics || !_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[_impalaDagree._metrics.nodes[id].fragment]) {
+      return;
+    }
+    var fragment = _impalaDagree._metrics.nodes[_impalaDagree._metrics.nodes[id].fragment];
+    return Object.keys(fragment.properties.hosts).reduce(function (previous, host) {
+      if (fragment.properties.hosts[host].TotalNetworkSendTime.value > previous.value) {
+        return fragment.properties.hosts[host].TotalNetworkSendTime;
+      } else {
+        return previous;
+      }
+    }, { value: -1, unit: 5 });
+  }
+
   function getTimelineData(key) {
   function getTimelineData(key) {
     if (!_impalaDagree._metrics) {
     if (!_impalaDagree._metrics) {
-      return [];
+      return;
     }
     }
-    var id = parseInt(key.split(':')[0], 10);
-    if (!_impalaDagree._metrics[id]) {
-      return [];
+    var id = getId(key);
+    if (!_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[id].timeline) {
+      return;
     }
     }
-    var times = _impalaDagree._metrics[id];
-    var timesKeys = Object.keys(times);
-    var timesKey;
-    for (var i = 0; i < timesKeys.length; i++) {
-      if (times[timesKeys[i]]['timeline'] && times[timesKeys[i]]['timeline']['Node Lifecycle Event Timeline']) {
-        timesKey = timesKeys[i];
-        break;
+    var timeline = _impalaDagree._metrics.nodes[id].timeline;
+    var times = Object.keys(timeline.hosts);
+    for (var i = 0; i < times.length; i++) {
+      if (!timeline.hosts[times[i]]['Node Lifecycle Event Timeline']) {
+        continue;
       }
       }
+      timeline.hosts[times[i]]['Node Lifecycle Event Timeline'].forEach(function (time, index, array) {
+        time.color = colors[index % colors.length];
+        return time;
+      });
     }
     }
-    if (!timesKey) {
-      return [];
-    }
-    var time = times[timesKey]['timeline']['Node Lifecycle Event Timeline'];
-    return time.map(function (time, index, array) {
-      var startTime = index > 0 && array[index - 1].value || 0;
-      return { starting_time: startTime, ending_time : time.value, duration: time.value - startTime, color: colors[index % colors.length], name: time.name, unit: time.unit };
-    });
+    return timeline;
   }
   }
 
 
   function renderTimeline(key) {
   function renderTimeline(key) {
     var datum = getTimelineData(key);
     var datum = getTimelineData(key);
-    if (!datum.length) {
+    if (!datum || !datum.hosts[datum.min] || !datum.hosts[datum.min]['Node Lifecycle Event Timeline']) {
       return '';
       return '';
     }
     }
     var end = _impalaDagree._metrics && _impalaDagree._metrics['max'] || 10;
     var end = _impalaDagree._metrics && _impalaDagree._metrics['max'] || 10;
     var divider = end > 33554428 ? 1000000 : 1; // values are in NS, scaling to MS as max pixel value is 33554428px ~9h in MS
     var divider = end > 33554428 ? 1000000 : 1; // values are in NS, scaling to MS as max pixel value is 33554428px ~9h in MS
     var html = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ' + (end / divider) + ' 10" class="timeline" preserveAspectRatio="none">';
     var html = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ' + (end / divider) + ' 10" class="timeline" preserveAspectRatio="none">';
-    html += datum.map(function(time, index) {
-      return '<rect x="' + (time.starting_time / divider) + '" width="' + (time.duration / divider)  + '" height="10" style="fill:' + time.color  +'"></rect>';
+    html += datum.hosts[datum.min]['Node Lifecycle Event Timeline'].map(function(time, index) {
+      return '<rect x="' + (time.start_time / divider) + '" width="' + (time.duration / divider)  + '" height="10" style="fill:' + time.color  +'"></rect>';
     }).join('');
     }).join('');
     html += '</svg>';
     html += '</svg>';
     return html;
     return html;
   }
   }
 
 
-  function renderTimelineLegend(key) {
-    var datum = getTimelineData(key);
-    if (!datum.length) {
-      return '';
-    }
-    return datum.map(function(time, index) {
-      return '<li><div class="legend-icon" style="background-color:' + time.color +' "></div><div class="metric-name">' + time.name + '</div> <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(time.duration, time.unit) + '</div></li>';
-    }).join('');
-  }
-
   function showDetail(id) {
   function showDetail(id) {
     var data;
     var data;
-    if (_impalaDagree._metrics[id] && _impalaDagree._metrics[id]['averaged']['metrics']) {
-      data = _impalaDagree._metrics[id]['averaged']['metrics'];
-    } else if (_impalaDagree._metrics[id] && _impalaDagree._metrics[id]['metrics']) {
-      data = _impalaDagree._metrics[id]['metrics'][Object.keys(_impalaDagree._metrics[id]['metrics'])[0]];
+    if (!_impalaDagree._metrics || !_impalaDagree._metrics.nodes[id]) {
+      return;
     }
     }
+    var data = _impalaDagree._metrics.nodes[id];
+
     d3.select('.query-plan').classed('open', true);
     d3.select('.query-plan').classed('open', true);
     var details = d3.select('.query-plan .details');
     var details = d3.select('.query-plan .details');
     var key = getKey(id);
     var key = getKey(id);
-    details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h3>' + states_by_name[key].label+ '</h3></div>')
+    details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h4>' + states_by_name[key].label+ '</h4></div>')
     var detailsContent = details.append('div').classed('details-content', true);
     var detailsContent = details.append('div').classed('details-content', true);
 
 
     var timeline = renderTimeline(key, '');
     var timeline = renderTimeline(key, '');
@@ -273,32 +287,69 @@ function impalaDagre(id) {
       var timelineSection = detailsContent.append('div').classed('details-section', true);
       var timelineSection = detailsContent.append('div').classed('details-section', true);
       var timelineTitle = timelineSection.append('header');
       var timelineTitle = timelineSection.append('header');
       timelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
       timelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
-      timelineTitle.append('h4').text(window.HUE_I18n.profile.timeline);
+      timelineTitle.append('h5').text(window.HUE_I18n.profile.timeline);
       timelineSection.node().appendChild($.parseXML(renderTimeline(key, '')).children[0]);
       timelineSection.node().appendChild($.parseXML(renderTimeline(key, '')).children[0]);
 
 
-      timelineSection.append('ol').classed('', true).html(renderTimelineLegend(key));
-      detailsContent.append('div').classed('divider', true);
+      var timelineSectionTable = timelineSection.append('table');
+      timelineSectionTable.append('thead').selectAll('tr').data(['\u00A0'].concat(Object.keys(_impalaDagree._metrics.nodes[id].timeline.hosts).sort())).enter().append('tr').append('td').text(function (host, i) { return i > 0 ? 'Host ' + i : host; }).attr('title', function (host) { return host; });
+      var timelineSectionTableBody = timelineSectionTable.append('tbody');
+      var timelineHosts = Object.keys(_impalaDagree._metrics.nodes[id].timeline.hosts).sort().map(function (host) { return _impalaDagree._metrics.nodes[id].timeline.hosts[host]; });
+      var timelineSectionTableCols = timelineSectionTableBody.selectAll('tr').data(timelineHosts);
+      var timelineSectionTableCol0 = timelineSectionTableBody.selectAll('tr').data(timelineHosts.slice(0,1));
+      timelineSectionTableCol0.enter().append('tr').selectAll('td').data(function (x) { return x['Node Lifecycle Event Timeline']; }).enter().append('td').html(function (time) { return '<div class="legend-icon" title="' + time.name + '" style="background-color:' + time.color +' "></div><div class="metric-name">' + time.name + '</div>'; });
+      timelineSectionTableCols.enter().append('tr').selectAll('td').data(function (x) { return x['Node Lifecycle Event Timeline']; }).enter().append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
     }
     }
 
 
     var metricsSection = detailsContent.append('div').classed('details-section', true);
     var metricsSection = detailsContent.append('div').classed('details-section', true);
+    var metricsChildSections = metricsSection.selectAll('div').data(Object.keys(data.children));
 
 
     var metricsTitle = metricsSection.append('header');
     var metricsTitle = metricsSection.append('header');
     metricsTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-bar-chart');
     metricsTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-bar-chart');
-    metricsTitle.append('h4').text(window.HUE_I18n.profile.metrics);
-
-    var metricsContent = metricsSection.append('ul').classed('metrics', true);
-
-    var metrics = metricsContent.selectAll('li')
-    .data(Object.keys(data).sort().map(function (key) { return data[key]; }));
-    metrics.exit().remove();
-    metrics.enter().append('li');
-    metrics.html(function (datum) { return '<div class="metric-name">' + datum.name + '</div> <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(datum.value, datum.unit) + '</div>'; });
+    metricsTitle.append('h5').text(window.HUE_I18n.profile.metrics);
+
+    var metricsContent = metricsSection.append('table').classed('metrics', true);
+    var metricsHosts = Object.keys(data.properties.hosts).sort().map(function (key) { return data.properties.hosts[key]; });
+    var metricsCols = metricsContent.selectAll('tr').data(metricsHosts);
+    var metricsCols0 = metricsContent.selectAll('tr').data(metricsHosts.slice(0,1));
+    metricsCols0.enter().append('tr').selectAll('td').data(function (host) { return Object.keys(host).sort(); }).enter().append('td').text(function (x) { return x; }).attr('title', function (x) { return x; });
+    metricsCols.enter().append('tr').selectAll('td').data(function (x) { return Object.keys(x).sort().map(function (key) {return x[key]; }) }).enter().append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.value, datum.unit); });
+    metricsContent.append('thead').selectAll('tr').data(['\u00A0'].concat(Object.keys(data.properties.hosts).sort())).enter().append('tr').append('td').text(function (x, i) { return i > 0 ? x === 'averaged' ? x : 'Host ' + (i - 1) : x; }).attr('title', function (x) {return x;});
+
+    var metricsChildSectionsContent = metricsChildSections.enter().append('div');
+    metricsChildSectionsContent.append('header').append('h5').text(function (key) { return key; });
+    var metricsChildSectionsContentTable = metricsChildSectionsContent.append('table').classed('metrics', true);
+    var fChildrenHosts = function (key) { return Object.keys(data.children[key].hosts).sort().map(function (host) { return data.children[key].hosts[host]; }); };
+    var metricsChildSectionsContentCols = metricsChildSectionsContentTable.selectAll('tr').data(function (key) { return fChildrenHosts(key); });
+    var metricsChildSectionsContentCols0 = metricsChildSectionsContentTable.selectAll('tr').data(function (key) { return fChildrenHosts(key).slice(0,1); });
+    metricsChildSectionsContentCols0.enter().append('tr').selectAll('td').data(function (host) { return Object.keys(host).sort(); }).enter().append('td').text(function (x) { return x; }).attr('title', function (x) { return x; });
+    metricsChildSectionsContentCols.enter().append('tr').selectAll('td').data(function (x) { return Object.keys(x).sort().map(function (key) {return x[key]; }) }).enter().append('td').text(function(datum) { return ko.bindingHandlers.numberFormat.human(datum.value, datum.unit);});
+    metricsChildSectionsContentTable.append('thead').selectAll('tr').data(function (key) { return ['\u00A0'].concat(Object.keys(data.children[key].hosts).sort()); }).enter().append('tr').append('td').text(function (x, i) { return i > 0 ? x === 'averaged' ? x : 'Host ' + (i - 1) : x; }).attr('title', function (x) {return x;});
   }
   }
 
 
   function hideDetail(id) {
   function hideDetail(id) {
     d3.select('.query-plan').classed('open', false);
     d3.select('.query-plan').classed('open', false);
   }
   }
 
 
+  function getProperty(object, path) {
+    var keys = path.split('.');
+    for (var i = 0; i < keys.length; i++) {
+      object = object[keys[i]];
+    }
+    return object;
+  }
+
+  function average(states, metric) {
+    var sum = 0;
+    for (var i = 0; i < states.length; i++) {
+      sum += getProperty(states[i], metric);
+    }
+    return states.length > 0 ? sum / states.length : 0;
+  }
+
+  function averageCombined(avg1, avg2, count1, count2) {
+    return (avg1 * count1 + avg2 * count2) / (count1 + count2);
+  }
+
   function renderGraph() {
   function renderGraph() {
     var plan = _impalaDagree._plan;
     var plan = _impalaDagree._plan;
     if (!plan || !plan.plan_nodes || !plan.plan_nodes.length) return;
     if (!plan || !plan.plan_nodes || !plan.plan_nodes.length) return;
@@ -309,43 +360,54 @@ function impalaDagre(id) {
     var max_node_time = 0;
     var max_node_time = 0;
     plan["plan_nodes"].forEach(function(parent) {
     plan["plan_nodes"].forEach(function(parent) {
       max_node_time = Math.max(
       max_node_time = Math.max(
-        build(parent, null, edges, states, colour_idx, max_node_time));
+        build(parent, null, edges, states, colour_idx, max_node_time, 1, 1));
       // Pick a new colour for each plan fragment
       // Pick a new colour for each plan fragment
       colour_idx = (colour_idx + 1) % colours.length;
       colour_idx = (colour_idx + 1) % colours.length;
     });
     });
-
+    var avgStates = average(states, 'max_time_val');
+    var edgesIO = edges.filter(function (edge) {
+      return edge.content.unit === 5;
+    });
+    var edgesNonIO = edges.filter(function (edge) {
+      return edge.content.unit === 0;
+    });
+    var avgEdgesIO = average(edgesIO, 'content.value');
+    var avgEdgesNonIO = average(edgesNonIO, 'content.value');
+    var avgCombined = averageCombined(avgStates, avgEdgesIO, states.length, edgesIO.length);
+    var avg = { '0': avgEdgesNonIO, '5': avgCombined};
     // Keep a map of names to states for use when processing edges.
     // Keep a map of names to states for use when processing edges.
     states.forEach(function(state) {
     states.forEach(function(state) {
       // Build the label for the node from the name and the detail
       // Build the label for the node from the name and the detail
-      var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + parseInt(state.name.split(':')[0], 10) + ");\">"; // TODO: Remove Hue dependency
-      html += getIcon(state.icon)
-      html += "<span class='name'>" + state.label + "</span><br/>";
-      html += "<span class='metric'>" + state.max_time + "</span>";
+      var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + getId(state.name) + ");\">"; // TODO: Remove Hue dependency
+      html += getIcon(state.icon);
+      html += "<span style='display: inline-block;'><span class='name'>" + state.label + "</span><br/>";
+      var aboveAverageClass = state.max_time_val > avg ? 'above-average' : '';
+      html += "<span class='metric " + aboveAverageClass + "'>" + state.max_time + "</span>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
-      html += "<span class='id'>" + state.name + "</span>";
+      if (state.predicates) {
+        html += "<span class='detail'>" + state.predicates + "</span><br/>";
+      }
+      html += "<span class='id'>" + state.name + "</span></span>";
       html += renderTimeline(state.name);
       html += renderTimeline(state.name);
       html += "</div>";
       html += "</div>";
 
 
       var style = state.style;
       var style = state.style;
 
 
-      // If colouring nodes by total time taken, choose a shade in the cols_by_time list
-      // with idx proportional to the max time of the node divided by the max time over all
-      // nodes.
-      /*if (document.getElementById("colour_scheme").checked) {
-        var idx = (cols_by_time.length - 1) * (state.max_time_val / (1.0 * max_node_time));
-        style = "fill: " + cols_by_time[Math.floor(idx)];
-      }*/
       g.setNode(state.name, { "label": html,
       g.setNode(state.name, { "label": html,
                               "labelType": "html",
                               "labelType": "html",
                               "style": style });
                               "style": style });
       states_by_name[state.name] = state;
       states_by_name[state.name] = state;
     });
     });
-
     edges.forEach(function(edge) {
     edges.forEach(function(edge) {
       // Impala marks 'broadcast' as a property of the receiver, not the sender. We use
       // Impala marks 'broadcast' as a property of the receiver, not the sender. We use
       // '(BCAST)' to denote that a node is duplicating its output to all receivers.
       // '(BCAST)' to denote that a node is duplicating its output to all receivers.
-      if (states_by_name[edge.end].is_broadcast) {
-        edge.style.label += " * " + states_by_name[edge.end].num_instances;
+      /*if (states_by_name[edge.end].is_broadcast) {
+        if (states_by_name[edge.end].num_instances > 1) {
+          edge.style.label += " * " + states_by_name[edge.end].num_instances;
+        }
+      }*/
+      if (edge.content.value > avg[edge.content.unit]) {
+        edge.style.labelStyle = "font-weight: bold";
       }
       }
       g.setEdge(edge.start, edge.end, edge.style);
       g.setEdge(edge.start, edge.end, edge.style);
     });
     });

+ 55 - 26
apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less

@@ -96,34 +96,34 @@
     .badge {
     .badge {
       color: @cui-gray-800;
       color: @cui-gray-800;
       text-shadow: none;
       text-shadow: none;
+      font-weight: normal;
     }
     }
     .metric {
     .metric {
       position: absolute;
       position: absolute;
       top: 0px;
       top: 0px;
       right: 0px;
       right: 0px;
-      font-weight: normal;
     }
     }
     .name {
     .name {
-      padding-right: 80px;
+      text-transform: capitalize;
+      font-size: 13px;
     }
     }
     .detail {
     .detail {
-      font-weight: normal;
       overflow: hidden;
       overflow: hidden;
       text-overflow: ellipsis;
       text-overflow: ellipsis;
-      width: calc(~"100% - 32px");
-      max-width: 185px;
+      width: 150px;
       display: inline-block;
       display: inline-block;
       height: 14px;
       height: 14px;
+      text-transform: lowercase;
     }
     }
     foreignObject {
     foreignObject {
       .fa {
       .fa {
         padding-top: 3px;
         padding-top: 3px;
         padding-right: 2px;
         padding-right: 2px;
-        float: left;
+        vertical-align: top;
       }
       }
       .hi {
       .hi {
         padding-right: 5px;
         padding-right: 5px;
-        float: left;
+        vertical-align: top;
       }
       }
     }
     }
     .fa {
     .fa {
@@ -153,7 +153,7 @@
       display: none;
       display: none;
     }
     }
     .node.active {
     .node.active {
-      rect {
+      > rect {
         filter: url(#dropshadow);
         filter: url(#dropshadow);
         stroke: @hue-primary-color-dark;
         stroke: @hue-primary-color-dark;
         fill: @hue-primary-color-light;
         fill: @hue-primary-color-light;
@@ -196,10 +196,15 @@
     }
     }
     .edgeLabel text {
     .edgeLabel text {
       font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
       font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+      fill: @cui-gray-800;
+      font-size: inherit;
     }
     }
     foreignObject > div {
     foreignObject > div {
       position: relative;
       position: relative;
     }
     }
+    .above-average {
+      font-weight: bold;
+    }
     .timeline {
     .timeline {
       width: 175px;
       width: 175px;
       height: 10px;
       height: 10px;
@@ -207,7 +212,7 @@
       border-radius: 4px;
       border-radius: 4px;
     }
     }
     .details {
     .details {
-      background-color: white;
+      background-color: @cui-white;
       box-shadow: 0px 0px 10px 0px;
       box-shadow: 0px 0px 10px 0px;
       color: @cui-gray-600;
       color: @cui-gray-600;
       position: absolute;
       position: absolute;
@@ -232,6 +237,10 @@
         margin: 0px;
         margin: 0px;
         display: inline-block;
         display: inline-block;
       }
       }
+      h5 {
+        margin: 0px;
+        display: inline-block;
+      }
       ol {
       ol {
         list-style-type: none;
         list-style-type: none;
         margin: 0px;
         margin: 0px;
@@ -244,6 +253,9 @@
       }
       }
       .details-section {
       .details-section {
         padding: 4px;
         padding: 4px;
+        header {
+          margin-top: 10px;
+        }
       }
       }
       .metric-title {
       .metric-title {
         background-color: @hue-primary-color-light;
         background-color: @hue-primary-color-light;
@@ -262,31 +274,48 @@
         overflow-y: scroll;
         overflow-y: scroll;
         max-height: calc(~"100% - 40px");
         max-height: calc(~"100% - 40px");
       }
       }
+      .legend-icon {
+        width: 8px;
+        height: 8px;
+        border-radius: 5px;
+        border: 1px solid @cui-gray-300;
+        display: inline-block;
+      }
       .metric-name {
       .metric-name {
-        color: @cui-gray-800;
-        width: 107px;
-        overflow: hidden;
-        text-align: right;
         display: inline-block;
         display: inline-block;
-        white-space: nowrap;
-        vertical-align: middle;
-        font-weight: bold;
         padding-left: 2px;
         padding-left: 2px;
       }
       }
-      .metric-value {
+      .details-section .timeline {
+        margin-bottom: 10px;
+      }
+      table {
+        display: table;
+      }
+      table tr {
+        display: table-cell;
+      }
+      table tr td {
         color: @cui-gray-800;
         color: @cui-gray-800;
+        max-width: 107px;
         overflow: hidden;
         overflow: hidden;
-        width: 60px;
-        vertical-align: middle;
-        display: inline-block;
+        display: block;
         white-space: nowrap;
         white-space: nowrap;
+        padding-right: 5px;
+        text-overflow: ellipsis;
       }
       }
-      .legend-icon {
-        width: 8px;
-        height: 8px;
-        border-radius: 5px;
-        border: 1px solid @cui-gray-300;
-        display: inline-block;
+      table thead tr td {
+        background-color: #FFFFFF;
+        border-bottom: 1px solid @cui-gray-300;
+        text-transform: capitalize;
+      }
+      table tr:nth-child(1) td {
+        border-right: 1px solid @cui-gray-300;
+      }
+      table thead tr td:nth-child(odd) {
+        background-color:@cui-white;
+      }
+      table tr td:nth-child(odd) {
+        background-color: @cui-gray-050;
       }
       }
     }
     }
   }
   }

+ 0 - 2
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -51,7 +51,6 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 </style>
 </style>
 % endif
 % endif
 
 
-<link rel="stylesheet" href="${ static('desktop/ext/css/c3.min.css') }">
 <link rel="stylesheet" href="${ static('jobbrowser/css/jobbrowser-embeddable.css') }">
 <link rel="stylesheet" href="${ static('jobbrowser/css/jobbrowser-embeddable.css') }">
 
 
 <script src="${ static('oozie/js/dashboard-utils.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('oozie/js/dashboard-utils.js') }" type="text/javascript" charset="utf-8"></script>
@@ -60,7 +59,6 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 <script src="${ static('desktop/ext/js/knockout-sortable.min.js') }"></script>
 <script src="${ static('desktop/ext/js/knockout-sortable.min.js') }"></script>
 <script src="${ static('desktop/js/ko.editable.js') }"></script>
 <script src="${ static('desktop/js/ko.editable.js') }"></script>
 <script src="${ static('desktop/ext/js/d3.v5.js') }"></script>
 <script src="${ static('desktop/ext/js/d3.v5.js') }"></script>
-<script src="${ static('desktop/ext/js/c3.min.js') }"></script>
 
 
 % if ENABLE_QUERY_BROWSER.get():
 % if ENABLE_QUERY_BROWSER.get():
 <script src="${ static('desktop/ext/js/d3.v3.js') }"></script>
 <script src="${ static('desktop/ext/js/d3.v3.js') }"></script>

File diff suppressed because it is too large
+ 0 - 0
desktop/core/src/desktop/static/desktop/ext/css/c3.min.css


File diff suppressed because it is too large
+ 0 - 1
desktop/core/src/desktop/static/desktop/ext/js/c3.min.js


+ 19 - 9
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -5936,6 +5936,12 @@
         editor.commands.off('afterExec', afterExecListener);
         editor.commands.off('afterExec', afterExecListener);
       });
       });
       editor.$blockScrolling = Infinity;
       editor.$blockScrolling = Infinity;
+
+      var range = options.highlightedRange ? options.highlightedRange() : null;
+      if (range && snippet.lastAceSelectionRowOffset()) {
+        var offset = snippet.lastAceSelectionRowOffset();
+        editor.selection.moveTo(range.start.row + offset, range.start.column);
+      }
       snippet.ace(editor);
       snippet.ace(editor);
     },
     },
 
 
@@ -5948,6 +5954,13 @@
           editor.setReadOnly(options.readOnly);
           editor.setReadOnly(options.readOnly);
         }
         }
         var range = options.highlightedRange ? options.highlightedRange() : null;
         var range = options.highlightedRange ? options.highlightedRange() : null;
+        if (editor.session.$backMarkers) {
+          for (var marker in editor.session.$backMarkers) {
+            if (editor.session.$backMarkers[marker].clazz === 'highlighted') {
+              editor.session.removeMarker(editor.session.$backMarkers[marker].id);
+            }
+          }
+        }
         editor.session.setMode(snippet.getAceMode());
         editor.session.setMode(snippet.getAceMode());
         if (range && JSON.stringify(range.start) !== JSON.stringify(range.end)) {
         if (range && JSON.stringify(range.start) !== JSON.stringify(range.end)) {
           var conflictingWithErrorMarkers = false;
           var conflictingWithErrorMarkers = false;
@@ -5959,9 +5972,6 @@
                   conflictingWithErrorMarkers = true;
                   conflictingWithErrorMarkers = true;
                 }
                 }
               }
               }
-              if (editor.session.$backMarkers[marker].clazz === 'highlighted') {
-                editor.session.removeMarker(editor.session.$backMarkers[marker].id);
-              }
             }
             }
           }
           }
           if (!conflictingWithErrorMarkers) {
           if (!conflictingWithErrorMarkers) {
@@ -7510,9 +7520,9 @@
       },
       },
       update: function (element, valueAccessor) {
       update: function (element, valueAccessor) {
         var props = ko.unwrap(valueAccessor());
         var props = ko.unwrap(valueAccessor());
-        this._impalaDagre.update(props.value);
         this._impalaDagre.metrics(props.metrics);
         this._impalaDagre.metrics(props.metrics);
         this._impalaDagre.height(props.height);
         this._impalaDagre.height(props.height);
+        this._impalaDagre.update(props.value);
       }
       }
     };
     };
   })();
   })();
@@ -7577,12 +7587,12 @@
         value = value * 1;
         value = value * 1;
         if (value < Math.pow(10, 3)) {
         if (value < Math.pow(10, 3)) {
           return value + " ns";
           return value + " ns";
-        } else if (value < Math.pow(10, 6)) {
+        } else if (value - Math.pow(10, 6) < -Math.pow(10, 3) / 2) { // Make sure rounding doesn't cause numbers to have more than 4 significant digits.
           value = (value * 1.0) / Math.pow(10, 3);
           value = (value * 1.0) / Math.pow(10, 3);
-          return sprintf("%.2f us", value);
-        } else if (value < Math.pow(10, 9)) {
+          return sprintf("%.1f us", value);
+        } else if (value - Math.pow(10, 9) < -Math.pow(10, 6) / 2) {
           value = (value * 1.0) / Math.pow(10, 6);
           value = (value * 1.0) / Math.pow(10, 6);
-          return sprintf("%.2f ms", value);
+          return sprintf("%.1f ms", value);
         } else {
         } else {
           // get the ms value
           // get the ms value
           var SECOND = 1;
           var SECOND = 1;
@@ -7602,7 +7612,7 @@
           }
           }
 
 
           if (value > SECOND) {
           if (value > SECOND) {
-            buffer += sprintf("%.2f s", value * 1.0 / SECOND);
+            buffer += sprintf("%.1f s", value * 1.0 / SECOND);
           }
           }
           return buffer;
           return buffer;
         }
         }

+ 11 - 1
desktop/core/src/desktop/templates/hue_icons.mako

@@ -315,7 +315,17 @@
     <symbol id="hi-sigma" viewBox="0 0 32 32">
     <symbol id="hi-sigma" viewBox="0 0 32 32">
       <path d="M29.425 22.96l1.387-2.96h1.188l-2 12h-30v-2.32l10.361-12.225-10.361-10.361v-7.094h30.625l1.375 8h-1.074l-0.585-1.215c-1.104-2.293-1.934-2.785-4.341-2.785h-20.688l11.033 11.033-9.294 10.967h16.949c3.625 0 4.583-1.299 5.425-3.040z"></path>
       <path d="M29.425 22.96l1.387-2.96h1.188l-2 12h-30v-2.32l10.361-12.225-10.361-10.361v-7.094h30.625l1.375 8h-1.074l-0.585-1.215c-1.104-2.293-1.934-2.785-4.341-2.785h-20.688l11.033 11.033-9.294 10.967h16.949c3.625 0 4.583-1.299 5.425-3.040z"></path>
     </symbol>
     </symbol>
-
+    <symbol id="hi-sigma-stream" viewBox="0 0 32 32">
+      <g>
+        <path d="m24.29172,19.54425l1.13561,-2.50675l0.97268,0l-1.6375,10.1625l-24.5625,0l0,-1.96475l8.48307,-10.35304l-8.48307,-8.77447l0,-6.00773l25.07422,0l1.12578,6.775l-0.87933,0l-0.47897,-1.02896c-0.9039,-1.94189 -1.58346,-2.35854 -3.55419,-2.35854l-16.9383,0l9.03327,9.34357l-7.60946,9.28768l13.87699,0c2.96797,0 3.75233,-1.10009 4.44172,-2.5745l-0.00001,-0.00001z"/>
+      </g>
+      <g>
+       <path d="m30.15,17l-12.10001,0c-0.91094,0 -1.65,0.69107 -1.65,1.54285l0,11.31428c0,0.85179 0.73906,1.54285 1.65,1.54285l12.10001,0c0.91094,0 1.65,-0.69107 1.65,-1.54285l0,-11.31428c0,-0.85179 -0.73906,-1.54285 -1.65,-1.54285z" fill="white"/>
+      </g>
+      <g transform="translate(4 4) scale(0.05 0.05)">
+       <path d="m255.68435,333.84284l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714zm293.53124,50.53571l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714zm-39.1375,111.17856l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714z"/>
+      </g>
+    </symbol>
     <symbol id="hi-unnest" viewBox="0 0 32 32">
     <symbol id="hi-unnest" viewBox="0 0 32 32">
       <path d="m30,8l2,0l0,16l-2,0l0,-16z"/>
       <path d="m30,8l2,0l0,16l-2,0l0,-16z"/>
       <path d="m7,14l22,0l0,4l-22,0l0,5l-7,-7l7,-7l0,5z"/>
       <path d="m7,14l22,0l0,4l-22,0l0,5l-7,-7l7,-7l0,5z"/>

+ 1 - 1
desktop/core/src/desktop/templates/ko_components/ko_drop_down.mako

@@ -39,7 +39,7 @@ from desktop.views import _ko
     <i class="fa fa-caret-down"></i>
     <i class="fa fa-caret-down"></i>
     <!-- /ko -->
     <!-- /ko -->
     <div class="hue-drop-down-container" data-bind="css: { 'open' : dropDownVisible, 'hue-drop-down-fixed': fixedPosition, 'hue-drop-down-container-searchable': searchable }, dropDownKeyUp: { onEsc: onEsc, onEnter: onEnter, dropDownVisible: dropDownVisible }">
     <div class="hue-drop-down-container" data-bind="css: { 'open' : dropDownVisible, 'hue-drop-down-fixed': fixedPosition, 'hue-drop-down-container-searchable': searchable }, dropDownKeyUp: { onEsc: onEsc, onEnter: onEnter, dropDownVisible: dropDownVisible }">
-      <div class="dropdown-menu" data-bind="visible: filteredEntries().length > 0, style: { 'overflow-y': !foreachVisible ? 'auto' : 'hidden' }">
+      <div style="overflow-y: auto;" class="dropdown-menu" data-bind="visible: filteredEntries().length > 0">
         <!-- ko if: foreachVisible -->
         <!-- ko if: foreachVisible -->
         <ul class="hue-inner-drop-down" data-bind="foreachVisible: { data: filteredEntries, minHeight: 34, container: '.dropdown-menu' }">
         <ul class="hue-inner-drop-down" data-bind="foreachVisible: { data: filteredEntries, minHeight: 34, container: '.dropdown-menu' }">
           <!-- ko if: typeof $data.divider !== 'undefined' && $data.divider -->
           <!-- ko if: typeof $data.divider !== 'undefined' && $data.divider -->

+ 1 - 1
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -2869,7 +2869,7 @@ ${ assist.assistPanel() }
           "source": ko.mapping.toJSON(self.source),
           "source": ko.mapping.toJSON(self.source),
           "destination": ko.mapping.toJSON(self.destination),
           "destination": ko.mapping.toJSON(self.destination),
           "start_time": ko.mapping.toJSON((new Date()).getTime()),
           "start_time": ko.mapping.toJSON((new Date()).getTime()),
-          "show_command": ko.mapping.toJSON(options.show || '')
+          "show_command": options.show || ''
         }, function (resp) {
         }, function (resp) {
           self.indexingStarted(false);
           self.indexingStarted(false);
           if (resp.status === 0) {
           if (resp.status === 0) {

+ 71 - 20
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -28,6 +28,7 @@ from thrift.transport import TTransport
 from libanalyze import dot
 from libanalyze import dot
 from libanalyze import gjson as jj
 from libanalyze import gjson as jj
 from libanalyze import models
 from libanalyze import models
+from libanalyze import utils
 from libanalyze.rules import to_double
 from libanalyze.rules import to_double
 
 
 
 
@@ -40,6 +41,7 @@ class Node(object):
     self.children = []
     self.children = []
     self.fragment = None
     self.fragment = None
     self.fragment_instance = None
     self.fragment_instance = None
+    self.plan_node = None
     self.pos = 0
     self.pos = 0
 
 
   def add_child(self, c):
   def add_child(self, c):
@@ -79,8 +81,8 @@ class Node(object):
 
 
   def is_regular(self):
   def is_regular(self):
     id = self.id()
     id = self.id()
-    matches = id and re.search(r'[a-zA-Z]+', id)
-    return id and matches is None
+    matches = id and re.search(r'^\d*$', id)
+    return id and matches
 
 
   def name(self):
   def name(self):
     matches = re.search(r'(.*?)(\s+\(((dst_)?id)=(\d+)\))?$', self.val.name)
     matches = re.search(r'(.*?)(\s+\(((dst_)?id)=(\d+)\))?$', self.val.name)
@@ -89,7 +91,11 @@ class Node(object):
     elif self.is_fragment():
     elif self.is_fragment():
       return re.search(r'(.*?Fragment) (F\d+)', self.val.name).group(1)
       return re.search(r'(.*?Fragment) (F\d+)', self.val.name).group(1)
     else:
     else:
-      return self.val.name
+      matches = re.search(r'(.*?)(\s+\(.*?\))?$', self.val.name)
+      if matches.group(2):
+        return matches.group(1)
+      else:
+        return self.val.name
 
 
   def id(self):
   def id(self):
     matches = re.search(r'(.*?)(\s+\(((dst_)?id)=(\d+)\))?$', self.val.name)
     matches = re.search(r'(.*?)(\s+\(((dst_)?id)=(\d+)\))?$', self.val.name)
@@ -128,17 +134,20 @@ class Node(object):
 
 
     return results
     return results
 
 
-  def foreach_lambda(self, method, fragment=None, fragment_instance=None, pos=0):
+  def foreach_lambda(self, method, plan_node=None, fragment=None, fragment_instance=None, pos=0):
     self.fragment = fragment
     self.fragment = fragment
     self.fragment_instance = fragment_instance
     self.fragment_instance = fragment_instance
     self.pos = pos
     self.pos = pos
+    self.plan_node = plan_node
     if self.is_fragment():
     if self.is_fragment():
       fragment = self
       fragment = self
     elif self.is_fragment_instance():
     elif self.is_fragment_instance():
       fragment_instance = self
       fragment_instance = self
+    elif self.is_plan_node():
+      plan_node = self
 
 
     for idx, x in enumerate(self.children):
     for idx, x in enumerate(self.children):
-      x.foreach_lambda(method, fragment=fragment, fragment_instance=fragment_instance, pos=idx)
+      x.foreach_lambda(method, plan_node=plan_node, fragment=fragment, fragment_instance=fragment_instance, pos=idx)
 
 
     method(self) # Post execution, because some results need child to have processed
     method(self) # Post execution, because some results need child to have processed
 
 
@@ -242,11 +251,15 @@ class Node(object):
     event_list = {}
     event_list = {}
     if self.val.event_sequences:
     if self.val.event_sequences:
       for s in self.val.event_sequences:
       for s in self.val.event_sequences:
+        start_time = 0
         sequence_name = s.name
         sequence_name = s.name
         event_list[sequence_name] = []
         event_list[sequence_name] = []
+        start_time = 0
         for i in range(len(s.labels)):
         for i in range(len(s.labels)):
+          event_duration = s.timestamps[i] - start_time
           event_name = s.labels[i]
           event_name = s.labels[i]
-          event_list[sequence_name].append({'name': event_name, 'value': s.timestamps[i], 'unit': 5})
+          event_list[sequence_name].append({'name': event_name, 'value': s.timestamps[i], 'unit': 5, 'start_time': start_time, 'duration': event_duration})
+          start_time = s.timestamps[i]
     return event_list
     return event_list
 
 
   def repr(self, indent):
   def repr(self, indent):
@@ -294,25 +307,63 @@ def summary(profile):
 
 
 def metrics(profile):
 def metrics(profile):
   execution_profile = profile.find_by_name('Execution Profile')
   execution_profile = profile.find_by_name('Execution Profile')
+  summary = profile.find_by_name("Summary")
+  plan_json = utils.parse_plan_details(summary.val.info_strings.get('Plan')) if summary.val.info_strings.get('Plan') else {}
   if not execution_profile:
   if not execution_profile:
     return {}
     return {}
-  counter_map = {'max': 0}
-  def get_metric(node, counter_map=counter_map):
-    if not node.is_plan_node():
-      return
-    nid = node.id()
-    if counter_map.get(nid) is None:
-      counter_map[nid] = {}
+  counter_map = {'nodes': {}, 'max': 0}
+  def flatten(node, counter_map=counter_map):
+    is_plan_node = node.is_plan_node()
+    is_parent_node = is_plan_node
+    if not is_plan_node:
+      if node.plan_node:
+        nid = node.plan_node.id()
+      elif node.is_fragment_instance():
+        is_parent_node = True
+        nid = node.fragment.id()
+      elif node.fragment:
+        nid = node.fragment.id()
+      else:
+        return
+    else:
+      nid = node.id()
+
     host = node.augmented_host()
     host = node.augmented_host()
+    metric_map = node.metric_map()
+    if counter_map['nodes'].get(nid) is None:
+      counter_map['nodes'][nid] = {'properties': { 'hosts': {} }, 'children': { }, 'timeline': {'hosts': {}}, 'other': {}}
+
     event_list = node.event_list();
     event_list = node.event_list();
-    if event_list and event_list.get('Node Lifecycle Event Timeline'):
-      last_value = event_list['Node Lifecycle Event Timeline'][len(event_list['Node Lifecycle Event Timeline']) - 1]['value']
-      counter_map['max'] = max(last_value, counter_map['max'])
-    if host:
-      counter_map[nid][host] = {'metrics': node.metric_map(), 'timeline': event_list}
+
+    if is_parent_node:
+      counter_map['nodes'][nid]['properties']['hosts'][host] = metric_map
+      if event_list:
+        counter_map['nodes'][nid]['timeline']['hosts'][host] = event_list
+      if plan_json.get(nid):
+        counter_map['nodes'][nid]['other'] = plan_json[nid]
+      if is_plan_node:
+        counter_map['nodes'][nid]['fragment'] = node.fragment.id()
     else:
     else:
-      counter_map[nid] = {'metrics': node.metric_map(), 'timeline': event_list}
-  execution_profile.foreach_lambda(get_metric)
+      name = node.name()
+      if counter_map['nodes'][nid]['children'].get(name) is None:
+        counter_map['nodes'][nid]['children'][name] = {'hosts': {}}
+      counter_map['nodes'][nid]['children'][name]['hosts'][host] = metric_map
+
+  execution_profile.foreach_lambda(flatten)
+
+  for nodeid, node in counter_map['nodes'].iteritems():
+    host_min = {'value': sys.maxint, 'host' : None}
+    for host_name, host_value in node['timeline']['hosts'].iteritems():
+      if host_value.get('Node Lifecycle Event Timeline'):
+        value = host_value['Node Lifecycle Event Timeline'][len(host_value['Node Lifecycle Event Timeline']) - 1]['value']
+        if value < host_min['value']:
+          host_min['value'] = value
+          host_min['host'] = host_name
+    node['timeline']['min'] = host_min.get('host', '')
+    if node['timeline']['min']:
+      node_min = node['timeline']['hosts'][node['timeline']['min']]['Node Lifecycle Event Timeline']
+      counter_map['max'] = max(node_min[len(node_min) - 1]['value'], counter_map['max'])
+
   counter_map['ImpalaServer'] = profile.find_by_name('ImpalaServer').metric_map()
   counter_map['ImpalaServer'] = profile.find_by_name('ImpalaServer').metric_map()
   return counter_map
   return counter_map
 
 

+ 9 - 6
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -589,12 +589,13 @@ class TopDownAnalysis:
         def add_host(node, exec_summary_json=exec_summary_json):
         def add_host(node, exec_summary_json=exec_summary_json):
           is_plan_node = node.is_plan_node()
           is_plan_node = node.is_plan_node()
           node_id = node.id()
           node_id = node.id()
+          nid = int(node_id) if node_id and node.is_regular() else -1
            # Setup Hosts & Broadcast
            # Setup Hosts & Broadcast
-          if node_id and node.is_regular() and int(node_id) in exec_summary_json:
-            exec_summary_node = exec_summary_json.get(int(node_id), {})
+          if node_id and node.is_regular() and nid in exec_summary_json:
+            exec_summary_node = exec_summary_json.get(nid, {})
             node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node.get('hosts', ''), unit=0))
             node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node.get('hosts', ''), unit=0))
             broadcast = 0
             broadcast = 0
-            if exec_summary_json[int(node_id)]["broadcast"]:
+            if exec_summary_json[nid]['broadcast']:
                 broadcast = 1
                 broadcast = 1
             node.val.counters.append(models.TCounter(name='Broadcast', value=broadcast, unit=0))
             node.val.counters.append(models.TCounter(name='Broadcast', value=broadcast, unit=0))
 
 
@@ -618,9 +619,11 @@ class TopDownAnalysis:
             # Make sure to substract the wait time for the exchange node
             # Make sure to substract the wait time for the exchange node
             if is_plan_node and re.search(r'EXCHANGE_NODE', node.val.name) is not None:
             if is_plan_node and re.search(r'EXCHANGE_NODE', node.val.name) is not None:
                 async_time = counter_map.get('AsyncTotalTime', models.TCounter(value=0)).value
                 async_time = counter_map.get('AsyncTotalTime', models.TCounter(value=0)).value
-                dequeue = node.find_by_name('Dequeue')
-                data_wait_time = dequeue.counter_map().get('DataWaitTime', models.TCounter(value=0)).value if dequeue else 0
-                local_time = counter_map['TotalTime'].value - counter_map['InactiveTotalTime'].value - async_time - data_wait_time
+                inactive_time = counter_map['InactiveTotalTime'].value
+                if inactive_time == 0:
+                  dequeue = node.find_by_name('Dequeue')
+                  inactive_time = dequeue.counter_map().get('DataWaitTime', models.TCounter(value=0)).value if dequeue else 0
+                local_time = counter_map['TotalTime'].value - inactive_time - async_time
 
 
             # For Hash Join, if the "LocalTime" metrics
             # For Hash Join, if the "LocalTime" metrics
             if is_plan_node and re.search(r'HASH_JOIN_NODE', node.val.name) is not None:
             if is_plan_node and re.search(r'HASH_JOIN_NODE', node.val.name) is not None:

+ 8 - 2
desktop/libs/libanalyze/src/libanalyze/utils.py

@@ -77,9 +77,15 @@ def parse_exec_summary(summary_string):
 def parse_plan_details(plan_string):
 def parse_plan_details(plan_string):
     """Given a query plan, extracts the query details per node"""
     """Given a query plan, extracts the query details per node"""
     result = {}
     result = {}
+    last_id = -1
     for line in plan_string.split("\n"):
     for line in plan_string.split("\n"):
-        match = re.search(r'^(?!F)[|-]?(\d+):.*?\[(.*?)\]', line.strip())
+        match = re.search(r'(?!F)[|-]?(\d+):.*?\[(.*?)\]', line.strip())
         if match:
         if match:
-            result[str(int(match.group(1)))] = match.group(2)
+          last_id = str(int(match.group(1)))
+          result[last_id] = {'detail': match.group(2)}
+        elif result.get(last_id):
+          match = re.search(r'[\|\s]*(.*?):\s?(.*)', line.strip())
+          if match:
+            result[last_id][match.group(1)] = match.group(2)
 
 
     return result
     return result

+ 77 - 19
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -52,7 +52,7 @@ var EditorViewModel = (function() {
   var Result = function (snippet, result) {
   var Result = function (snippet, result) {
     var self = this;
     var self = this;
 
 
-    snippet = $.extend(snippet, snippet.chartType == 'lines' && { // Retire line chart
+    $.extend(snippet, snippet.chartType == 'lines' && { // Retire line chart
         chartType: 'bars',
         chartType: 'bars',
         chartTimelineType: 'line'
         chartTimelineType: 'line'
     });
     });
@@ -97,7 +97,9 @@ var EditorViewModel = (function() {
         column: 0
         column: 0
       }
       }
     });
     });
-    self.statements_count = ko.observable(typeof result.statements_count != "undefined" && result.statements_count != null ? result.statements_count : 1);
+    // We don't keep track of any previous selection so prevent entering into batch execution mode after load by setting
+    // statements_count to 1. For the case when a selection is not starting at row 0.
+    self.statements_count = ko.observable(1);
     self.previous_statement_hash = ko.observable(typeof result.previous_statement_hash != "undefined" && result.previous_statement_hash != null ? result.previous_statement_hash : null);
     self.previous_statement_hash = ko.observable(typeof result.previous_statement_hash != "undefined" && result.previous_statement_hash != null ? result.previous_statement_hash : null);
     self.cleanedMeta = ko.computed(function () {
     self.cleanedMeta = ko.computed(function () {
       return ko.utils.arrayFilter(self.meta(), function (item) {
       return ko.utils.arrayFilter(self.meta(), function (item) {
@@ -214,6 +216,16 @@ var EditorViewModel = (function() {
     self.clear = function () {
     self.clear = function () {
       self.fetchedOnce(false);
       self.fetchedOnce(false);
       self.hasMore(false);
       self.hasMore(false);
+      self.statement_range({
+        start: {
+          row: 0,
+          column: 0
+        },
+        end: {
+          row: 0,
+          column: 0
+        }
+      });
       self.meta.removeAll();
       self.meta.removeAll();
       self.data.removeAll();
       self.data.removeAll();
       self.images.removeAll();
       self.images.removeAll();
@@ -232,6 +244,33 @@ var EditorViewModel = (function() {
     };
     };
   };
   };
 
 
+  Result.prototype.cancelBatchExecution = function () {
+    var self = this;
+    self.statements_count(1);
+    self.hasMore(false);
+    self.statement_range({
+      start: {
+        row: 0,
+        column: 0
+      },
+      end: {
+        row: 0,
+        column: 0
+      }
+    });
+    self.handle()['statement_id'] = 0;
+    self.handle()['start'] = {
+      row: 0,
+      column: 0
+    };
+    self.handle()['end'] = {
+      row: 0,
+      column: 0
+    };
+    self.handle()['has_more_statements'] = false;
+    self.handle()['previous_statement_hash'] = '';
+  };
+
   var getDefaultSnippetProperties = function (snippetType) {
   var getDefaultSnippetProperties = function (snippetType) {
     var properties = {};
     var properties = {};
 
 
@@ -958,7 +997,12 @@ var EditorViewModel = (function() {
       if (self.result.handle() && self.result.handle().has_more_statements) {
       if (self.result.handle() && self.result.handle().has_more_statements) {
         window.clearTimeout(executeNextTimeout);
         window.clearTimeout(executeNextTimeout);
         executeNextTimeout = setTimeout(function () {
         executeNextTimeout = setTimeout(function () {
-          self.execute(true); // Execute next, need to wait as we disabled fast click
+          // Prevent execution when statement selection has changed
+          if (self.lastExecutedStatements === self.statement()) {
+            self.execute(true); // Execute next, need to wait as we disabled fast click
+          } else {
+            self.result.cancelBatchExecution();
+          }
         }, 1000);
         }, 1000);
       }
       }
       if (self.lastExecutedStatement() && /CREATE|DROP/i.test(self.lastExecutedStatement().firstToken)) {
       if (self.lastExecutedStatement() && /CREATE|DROP/i.test(self.lastExecutedStatement().firstToken)) {
@@ -1465,11 +1509,34 @@ var EditorViewModel = (function() {
       self.showLongOperationWarning(false);
       self.showLongOperationWarning(false);
     }
     }
 
 
+    self.lastExecutedStatements = undefined;
+
     self.execute = function (automaticallyTriggered) {
     self.execute = function (automaticallyTriggered) {
-      var now = (new Date()).getTime(); // We don't allow fast clicks
-      if (!automaticallyTriggered && (self.status() === 'running' || self.status() === 'loading')) { // Do not cancel statements that are parts of a set of steps to execute (e.g. import). Only cancel statements as requested by user
-        self.cancel();
-      } else if (now - self.lastExecuted() < 1000 || ! self.isReady()) {
+      var now = (new Date()).getTime();
+      if (now - self.lastExecuted() < 1000 || ! self.isReady()) {
+        return; // Prevent fast clicks
+      }
+
+      if (!automaticallyTriggered) {
+        // Do not cancel statements that are parts of a set of steps to execute (e.g. import). Only cancel statements as requested by user
+        if (self.status() === 'running' || self.status() === 'loading') {
+          self.cancel(); // TODO: Wait for cancel to finish
+        } else {
+          self.result.clear()
+        }
+      }
+
+      if (self.editorMode() && self.result.statements_count() > 1 && self.lastExecutedStatements !== self.statement()) {
+        self.lastExecutedStatements = self.statement();
+        if (automaticallyTriggered) {
+          if (self.executingBlockingOperation) {
+            self.executingBlockingOperation.abort();
+            self.executingBlockingOperation = null;
+          }
+          self.result.cancelBatchExecution();
+        } else {
+          self.reexecute();
+        }
         return;
         return;
       }
       }
 
 
@@ -1485,6 +1552,8 @@ var EditorViewModel = (function() {
         huePubSub.publish('editor.refresh.statement.locations', self);
         huePubSub.publish('editor.refresh.statement.locations', self);
       }
       }
 
 
+      self.lastExecutedStatements = self.statement();
+
       if (self.ace()) {
       if (self.ace()) {
         huePubSub.publish('ace.set.autoexpand', { autoExpand: false, snippet: self });
         huePubSub.publish('ace.set.autoexpand', { autoExpand: false, snippet: self });
         var selectionRange = self.ace().getSelectionRange();
         var selectionRange = self.ace().getSelectionRange();
@@ -1628,18 +1697,7 @@ var EditorViewModel = (function() {
     };
     };
 
 
     self.reexecute = function () {
     self.reexecute = function () {
-      self.result.handle()['statement_id'] = 0;
-      self.result.handle()['start'] = {
-        row: 0,
-        column: 0
-      };
-      self.result.handle()['end'] = {
-        row: 0,
-        column: 0
-      };
-      self.result.handle()['has_more_statements'] = false;
-      self.result.handle()['previous_statement_hash'] = '';
-
+      self.result.cancelBatchExecution();
       self.execute();
       self.execute();
     };
     };
 
 

+ 1 - 1
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1823,7 +1823,7 @@ ${ sqlSyntaxDropdown.sqlSyntaxDropdown() }
       <i class="fa fa-fw fa-spinner fa-spin"></i>
       <i class="fa fa-fw fa-spinner fa-spin"></i>
     </a>
     </a>
     <!-- /ko -->
     <!-- /ko -->
-    <a class="snippet-side-btn" data-bind="click: reexecute, visible: $root.editorMode() && result.handle() && result.handle().has_more_statements, css: {'blue': $parent.history().length == 0 || $root.editorMode(), 'disabled': ! isReady() }" title="${ _('Restart from the first statement') }">
+    <a class="snippet-side-btn" data-bind="click: reexecute, visible: $root.editorMode() && result.statements_count() > 1, css: {'blue': $parent.history().length == 0 || $root.editorMode(), 'disabled': ! isReady() }" title="${ _('Restart from the first statement') }">
       <i class="fa fa-fw fa-repeat snippet-side-single"></i>
       <i class="fa fa-fw fa-repeat snippet-side-single"></i>
     </a>
     </a>
     <div class="label label-info" data-bind="attr: {'title':'${ _ko('Showing results of the statement #')}' + (result.statement_id() + 1)}, visible: $root.editorMode() && result.statements_count() > 1">
     <div class="label label-info" data-bind="attr: {'title':'${ _ko('Showing results of the statement #')}' + (result.statement_id() + 1)}, visible: $root.editorMode() && result.statements_count() > 1">

+ 1 - 1
ext/thirdparty/README.md

@@ -119,7 +119,7 @@ Checked-in frontend third party dependencies
 |classList.js|?|Unlicense|https://github.com/eligrey/classList.js|
 |classList.js|?|Unlicense|https://github.com/eligrey/classList.js|
 |clipboard.js|1.7.1|MIT|https://zenorocha.github.io/clipboard.js|
 |clipboard.js|1.7.1|MIT|https://zenorocha.github.io/clipboard.js|
 |Codemirror|3.11|MIT|https://github.com/codemirror/CodeMirror|
 |Codemirror|3.11|MIT|https://github.com/codemirror/CodeMirror|
-|D3|3.5.2+4.4.4|BSD|https://github.com/d3/d3|
+|D3|3.5.2,4.4.4,5.7.0|BSD|https://github.com/d3/d3|
 |Dropzone.js|4.3.0|MIT|https://github.com/enyo/dropzone|
 |Dropzone.js|4.3.0|MIT|https://github.com/enyo/dropzone|
 |django-debug-panel|0.8.3|BSD|https://pypi.org/project/django-debug-panel/0.8.3/|
 |django-debug-panel|0.8.3|BSD|https://pypi.org/project/django-debug-panel/0.8.3/|
 |django-debug-toolbar|1.3.2|BSD|https://pypi.python.org/pypi/django-debug-toolbar/1.3.2|
 |django-debug-toolbar|1.3.2|BSD|https://pypi.python.org/pypi/django-debug-toolbar/1.3.2|

Some files were not shown because too many files changed in this diff