|
|
@@ -24,7 +24,6 @@ function impalaDagre(id) {
|
|
|
},
|
|
|
metrics: function(data) {
|
|
|
_impalaDagree._metrics = data;
|
|
|
- renderGraph()
|
|
|
},
|
|
|
update: function(plan) {
|
|
|
_impalaDagree._plan = plan;
|
|
|
@@ -110,38 +109,49 @@ function impalaDagre(id) {
|
|
|
"#991F00", "#B22400", "#CC2900", "#E62E00", "#FF3300", "#FF4719"];
|
|
|
|
|
|
// Recursively build a list of edges and states that comprise the plan graph
|
|
|
- function build(node, parent, edges, states, colour_idx, max_node_time) {
|
|
|
+ function build(node, parent, edges, states, colour_idx, max_node_time, index, count) {
|
|
|
if (node["output_card"] === null || node["output_card"] === undefined) {
|
|
|
return;
|
|
|
}
|
|
|
+ var id = getId(node["label"]);
|
|
|
+ var metric_node = _impalaDagree._metrics && _impalaDagree._metrics.nodes[id]
|
|
|
+ var predicates = metric_node && (metric_node.other['group by'] || metric_node.other['hash predicates'] || metric_node.other['predicates']) || '';
|
|
|
states.push({ "name": node["label"],
|
|
|
"type": node["type"],
|
|
|
"label": node["name"],
|
|
|
"detail": node["label_detail"],
|
|
|
+ "predicates": predicates,
|
|
|
"num_instances": node["num_instances"],
|
|
|
"num_active": node["num_active"],
|
|
|
- "max_time": node["max_time"],
|
|
|
+ "max_time": ko.bindingHandlers.numberFormat.human(node["max_time_val"], 5),
|
|
|
"avg_time": node["avg_time"],
|
|
|
"icon": node["icon"],
|
|
|
"is_broadcast": node["is_broadcast"],
|
|
|
- "max_time_val": node["max_time_val"]});
|
|
|
+ "max_time_val": node["max_time_val"],
|
|
|
+ "width": "200px"});
|
|
|
+ var edgeCount;
|
|
|
if (parent) {
|
|
|
- var label_val = "" + ko.bindingHandlers.simplesize.humanSize(parseInt(node["output_card"], 10));
|
|
|
- edges.push({ start: node["label"], end: parent,
|
|
|
- style: { label: label_val }});
|
|
|
+ edgeCount = parseInt(node["output_card"], 10);
|
|
|
+ var label_val = "" + ko.bindingHandlers.simplesize.humanSize(edgeCount);
|
|
|
+ edges.push({ start: node["label"], end: parent, style: { label: label_val, labelpos: index === 0 && count > 1 ? 'l' : 'r' }, content: { value: edgeCount, unit: 0 } });
|
|
|
}
|
|
|
// Add an inter-fragment edge. We use a red dashed line to show that rows are crossing
|
|
|
// the fragment boundary.
|
|
|
if (node["data_stream_target"]) {
|
|
|
+ edgeCount = parseInt(node["output_card"], 10);
|
|
|
+ var sendTime = getMaxTotalNetworkSendTime(node["label"]);
|
|
|
+ var text = sendTime && ko.bindingHandlers.numberFormat.human(sendTime.value, sendTime.unit) || ko.bindingHandlers.simplesize.humanSize(edgeCount);
|
|
|
edges.push({ "start": node["label"],
|
|
|
"end": node["data_stream_target"],
|
|
|
- "style": { label: ko.bindingHandlers.simplesize.humanSize(parseInt(node["output_card"], 10)),
|
|
|
- style: "stroke-dasharray: 5, 5;"}});
|
|
|
+ "content": sendTime ? sendTime : { value: edgeCount, unit: 0 },
|
|
|
+ "style": { label: text,
|
|
|
+ style: "stroke-dasharray: 5, 5;",
|
|
|
+ labelpos: index === 0 && count > 1 ? 'l' : 'r' }});
|
|
|
}
|
|
|
max_node_time = Math.max(node["max_time_val"], max_node_time)
|
|
|
for (var i = 0; i < node["children"].length; ++i) {
|
|
|
max_node_time = build(
|
|
|
- node["children"][i], node["label"], edges, states, colour_idx, max_node_time);
|
|
|
+ node["children"][i], node["label"], edges, states, colour_idx, max_node_time, i, node["children"].length);
|
|
|
}
|
|
|
return max_node_time;
|
|
|
}
|
|
|
@@ -162,13 +172,17 @@ function impalaDagre(id) {
|
|
|
$("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
|
|
|
}
|
|
|
|
|
|
+ function getId(key) {
|
|
|
+ return parseInt(key.split(':')[0], 10);
|
|
|
+ }
|
|
|
+
|
|
|
function getKey(node) {
|
|
|
var nodes = g.nodes();
|
|
|
var key;
|
|
|
var nNode = parseInt(node, 10);
|
|
|
var keys = Object.keys(nodes);
|
|
|
for (var i = 0; i < keys.length; i++) {
|
|
|
- if (parseInt(nodes[keys[i]].split(':')[0], 10) == nNode) {
|
|
|
+ if (getId(nodes[keys[i]]) == nNode) {
|
|
|
key = nodes[keys[i]];
|
|
|
break;
|
|
|
}
|
|
|
@@ -203,69 +217,69 @@ function impalaDagre(id) {
|
|
|
return html;
|
|
|
}
|
|
|
|
|
|
+ function getMaxTotalNetworkSendTime(node) {
|
|
|
+ var id = getId(node);
|
|
|
+ if (!_impalaDagree._metrics || !_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[_impalaDagree._metrics.nodes[id].fragment]) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ var fragment = _impalaDagree._metrics.nodes[_impalaDagree._metrics.nodes[id].fragment];
|
|
|
+ return Object.keys(fragment.properties.hosts).reduce(function (previous, host) {
|
|
|
+ if (fragment.properties.hosts[host].TotalNetworkSendTime.value > previous.value) {
|
|
|
+ return fragment.properties.hosts[host].TotalNetworkSendTime;
|
|
|
+ } else {
|
|
|
+ return previous;
|
|
|
+ }
|
|
|
+ }, { value: -1, unit: 5 });
|
|
|
+ }
|
|
|
+
|
|
|
function getTimelineData(key) {
|
|
|
if (!_impalaDagree._metrics) {
|
|
|
- return [];
|
|
|
+ return;
|
|
|
}
|
|
|
- var id = parseInt(key.split(':')[0], 10);
|
|
|
- if (!_impalaDagree._metrics[id]) {
|
|
|
- return [];
|
|
|
+ var id = getId(key);
|
|
|
+ if (!_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[id].timeline) {
|
|
|
+ return;
|
|
|
}
|
|
|
- var times = _impalaDagree._metrics[id];
|
|
|
- var timesKeys = Object.keys(times);
|
|
|
- var timesKey;
|
|
|
- for (var i = 0; i < timesKeys.length; i++) {
|
|
|
- if (times[timesKeys[i]]['timeline'] && times[timesKeys[i]]['timeline']['Node Lifecycle Event Timeline']) {
|
|
|
- timesKey = timesKeys[i];
|
|
|
- break;
|
|
|
+ var timeline = _impalaDagree._metrics.nodes[id].timeline;
|
|
|
+ var times = Object.keys(timeline.hosts);
|
|
|
+ for (var i = 0; i < times.length; i++) {
|
|
|
+ if (!timeline.hosts[times[i]]['Node Lifecycle Event Timeline']) {
|
|
|
+ continue;
|
|
|
}
|
|
|
+ timeline.hosts[times[i]]['Node Lifecycle Event Timeline'].forEach(function (time, index, array) {
|
|
|
+ time.color = colors[index % colors.length];
|
|
|
+ return time;
|
|
|
+ });
|
|
|
}
|
|
|
- if (!timesKey) {
|
|
|
- return [];
|
|
|
- }
|
|
|
- var time = times[timesKey]['timeline']['Node Lifecycle Event Timeline'];
|
|
|
- return time.map(function (time, index, array) {
|
|
|
- var startTime = index > 0 && array[index - 1].value || 0;
|
|
|
- return { starting_time: startTime, ending_time : time.value, duration: time.value - startTime, color: colors[index % colors.length], name: time.name, unit: time.unit };
|
|
|
- });
|
|
|
+ return timeline;
|
|
|
}
|
|
|
|
|
|
function renderTimeline(key) {
|
|
|
var datum = getTimelineData(key);
|
|
|
- if (!datum.length) {
|
|
|
+ if (!datum || !datum.hosts[datum.min] || !datum.hosts[datum.min]['Node Lifecycle Event Timeline']) {
|
|
|
return '';
|
|
|
}
|
|
|
var end = _impalaDagree._metrics && _impalaDagree._metrics['max'] || 10;
|
|
|
var divider = end > 33554428 ? 1000000 : 1; // values are in NS, scaling to MS as max pixel value is 33554428px ~9h in MS
|
|
|
var html = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ' + (end / divider) + ' 10" class="timeline" preserveAspectRatio="none">';
|
|
|
- html += datum.map(function(time, index) {
|
|
|
- return '<rect x="' + (time.starting_time / divider) + '" width="' + (time.duration / divider) + '" height="10" style="fill:' + time.color +'"></rect>';
|
|
|
+ html += datum.hosts[datum.min]['Node Lifecycle Event Timeline'].map(function(time, index) {
|
|
|
+ return '<rect x="' + (time.start_time / divider) + '" width="' + (time.duration / divider) + '" height="10" style="fill:' + time.color +'"></rect>';
|
|
|
}).join('');
|
|
|
html += '</svg>';
|
|
|
return html;
|
|
|
}
|
|
|
|
|
|
- function renderTimelineLegend(key) {
|
|
|
- var datum = getTimelineData(key);
|
|
|
- if (!datum.length) {
|
|
|
- return '';
|
|
|
- }
|
|
|
- return datum.map(function(time, index) {
|
|
|
- return '<li><div class="legend-icon" style="background-color:' + time.color +' "></div><div class="metric-name">' + time.name + '</div> <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(time.duration, time.unit) + '</div></li>';
|
|
|
- }).join('');
|
|
|
- }
|
|
|
-
|
|
|
function showDetail(id) {
|
|
|
var data;
|
|
|
- if (_impalaDagree._metrics[id] && _impalaDagree._metrics[id]['averaged']['metrics']) {
|
|
|
- data = _impalaDagree._metrics[id]['averaged']['metrics'];
|
|
|
- } else if (_impalaDagree._metrics[id] && _impalaDagree._metrics[id]['metrics']) {
|
|
|
- data = _impalaDagree._metrics[id]['metrics'][Object.keys(_impalaDagree._metrics[id]['metrics'])[0]];
|
|
|
+ if (!_impalaDagree._metrics || !_impalaDagree._metrics.nodes[id]) {
|
|
|
+ return;
|
|
|
}
|
|
|
+ var data = _impalaDagree._metrics.nodes[id];
|
|
|
+
|
|
|
d3.select('.query-plan').classed('open', true);
|
|
|
var details = d3.select('.query-plan .details');
|
|
|
var key = getKey(id);
|
|
|
- details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h3>' + states_by_name[key].label+ '</h3></div>')
|
|
|
+ details.html('<header class="metric-title">' + getIcon(states_by_name[key].icon) + '<h4>' + states_by_name[key].label+ '</h4></div>')
|
|
|
var detailsContent = details.append('div').classed('details-content', true);
|
|
|
|
|
|
var timeline = renderTimeline(key, '');
|
|
|
@@ -273,32 +287,69 @@ function impalaDagre(id) {
|
|
|
var timelineSection = detailsContent.append('div').classed('details-section', true);
|
|
|
var timelineTitle = timelineSection.append('header');
|
|
|
timelineTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-access-time');
|
|
|
- timelineTitle.append('h4').text(window.HUE_I18n.profile.timeline);
|
|
|
+ timelineTitle.append('h5').text(window.HUE_I18n.profile.timeline);
|
|
|
timelineSection.node().appendChild($.parseXML(renderTimeline(key, '')).children[0]);
|
|
|
|
|
|
- timelineSection.append('ol').classed('', true).html(renderTimelineLegend(key));
|
|
|
- detailsContent.append('div').classed('divider', true);
|
|
|
+ var timelineSectionTable = timelineSection.append('table');
|
|
|
+ timelineSectionTable.append('thead').selectAll('tr').data(['\u00A0'].concat(Object.keys(_impalaDagree._metrics.nodes[id].timeline.hosts).sort())).enter().append('tr').append('td').text(function (host, i) { return i > 0 ? 'Host ' + i : host; }).attr('title', function (host) { return host; });
|
|
|
+ var timelineSectionTableBody = timelineSectionTable.append('tbody');
|
|
|
+ var timelineHosts = Object.keys(_impalaDagree._metrics.nodes[id].timeline.hosts).sort().map(function (host) { return _impalaDagree._metrics.nodes[id].timeline.hosts[host]; });
|
|
|
+ var timelineSectionTableCols = timelineSectionTableBody.selectAll('tr').data(timelineHosts);
|
|
|
+ var timelineSectionTableCol0 = timelineSectionTableBody.selectAll('tr').data(timelineHosts.slice(0,1));
|
|
|
+ timelineSectionTableCol0.enter().append('tr').selectAll('td').data(function (x) { return x['Node Lifecycle Event Timeline']; }).enter().append('td').html(function (time) { return '<div class="legend-icon" title="' + time.name + '" style="background-color:' + time.color +' "></div><div class="metric-name">' + time.name + '</div>'; });
|
|
|
+ timelineSectionTableCols.enter().append('tr').selectAll('td').data(function (x) { return x['Node Lifecycle Event Timeline']; }).enter().append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.duration, datum.unit); });
|
|
|
}
|
|
|
|
|
|
var metricsSection = detailsContent.append('div').classed('details-section', true);
|
|
|
+ var metricsChildSections = metricsSection.selectAll('div').data(Object.keys(data.children));
|
|
|
|
|
|
var metricsTitle = metricsSection.append('header');
|
|
|
metricsTitle.append('svg').classed('hi', true).append('use').attr('xlink:href', '#hi-bar-chart');
|
|
|
- metricsTitle.append('h4').text(window.HUE_I18n.profile.metrics);
|
|
|
-
|
|
|
- var metricsContent = metricsSection.append('ul').classed('metrics', true);
|
|
|
-
|
|
|
- var metrics = metricsContent.selectAll('li')
|
|
|
- .data(Object.keys(data).sort().map(function (key) { return data[key]; }));
|
|
|
- metrics.exit().remove();
|
|
|
- metrics.enter().append('li');
|
|
|
- metrics.html(function (datum) { return '<div class="metric-name">' + datum.name + '</div> <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(datum.value, datum.unit) + '</div>'; });
|
|
|
+ metricsTitle.append('h5').text(window.HUE_I18n.profile.metrics);
|
|
|
+
|
|
|
+ var metricsContent = metricsSection.append('table').classed('metrics', true);
|
|
|
+ var metricsHosts = Object.keys(data.properties.hosts).sort().map(function (key) { return data.properties.hosts[key]; });
|
|
|
+ var metricsCols = metricsContent.selectAll('tr').data(metricsHosts);
|
|
|
+ var metricsCols0 = metricsContent.selectAll('tr').data(metricsHosts.slice(0,1));
|
|
|
+ metricsCols0.enter().append('tr').selectAll('td').data(function (host) { return Object.keys(host).sort(); }).enter().append('td').text(function (x) { return x; }).attr('title', function (x) { return x; });
|
|
|
+ metricsCols.enter().append('tr').selectAll('td').data(function (x) { return Object.keys(x).sort().map(function (key) {return x[key]; }) }).enter().append('td').text(function (datum) { return ko.bindingHandlers.numberFormat.human(datum.value, datum.unit); });
|
|
|
+ metricsContent.append('thead').selectAll('tr').data(['\u00A0'].concat(Object.keys(data.properties.hosts).sort())).enter().append('tr').append('td').text(function (x, i) { return i > 0 ? x === 'averaged' ? x : 'Host ' + (i - 1) : x; }).attr('title', function (x) {return x;});
|
|
|
+
|
|
|
+ var metricsChildSectionsContent = metricsChildSections.enter().append('div');
|
|
|
+ metricsChildSectionsContent.append('header').append('h5').text(function (key) { return key; });
|
|
|
+ var metricsChildSectionsContentTable = metricsChildSectionsContent.append('table').classed('metrics', true);
|
|
|
+ var fChildrenHosts = function (key) { return Object.keys(data.children[key].hosts).sort().map(function (host) { return data.children[key].hosts[host]; }); };
|
|
|
+ var metricsChildSectionsContentCols = metricsChildSectionsContentTable.selectAll('tr').data(function (key) { return fChildrenHosts(key); });
|
|
|
+ var metricsChildSectionsContentCols0 = metricsChildSectionsContentTable.selectAll('tr').data(function (key) { return fChildrenHosts(key).slice(0,1); });
|
|
|
+ metricsChildSectionsContentCols0.enter().append('tr').selectAll('td').data(function (host) { return Object.keys(host).sort(); }).enter().append('td').text(function (x) { return x; }).attr('title', function (x) { return x; });
|
|
|
+ metricsChildSectionsContentCols.enter().append('tr').selectAll('td').data(function (x) { return Object.keys(x).sort().map(function (key) {return x[key]; }) }).enter().append('td').text(function(datum) { return ko.bindingHandlers.numberFormat.human(datum.value, datum.unit);});
|
|
|
+ metricsChildSectionsContentTable.append('thead').selectAll('tr').data(function (key) { return ['\u00A0'].concat(Object.keys(data.children[key].hosts).sort()); }).enter().append('tr').append('td').text(function (x, i) { return i > 0 ? x === 'averaged' ? x : 'Host ' + (i - 1) : x; }).attr('title', function (x) {return x;});
|
|
|
}
|
|
|
|
|
|
function hideDetail(id) {
|
|
|
d3.select('.query-plan').classed('open', false);
|
|
|
}
|
|
|
|
|
|
+ function getProperty(object, path) {
|
|
|
+ var keys = path.split('.');
|
|
|
+ for (var i = 0; i < keys.length; i++) {
|
|
|
+ object = object[keys[i]];
|
|
|
+ }
|
|
|
+ return object;
|
|
|
+ }
|
|
|
+
|
|
|
+ function average(states, metric) {
|
|
|
+ var sum = 0;
|
|
|
+ for (var i = 0; i < states.length; i++) {
|
|
|
+ sum += getProperty(states[i], metric);
|
|
|
+ }
|
|
|
+ return states.length > 0 ? sum / states.length : 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ function averageCombined(avg1, avg2, count1, count2) {
|
|
|
+ return (avg1 * count1 + avg2 * count2) / (count1 + count2);
|
|
|
+ }
|
|
|
+
|
|
|
function renderGraph() {
|
|
|
var plan = _impalaDagree._plan;
|
|
|
if (!plan || !plan.plan_nodes || !plan.plan_nodes.length) return;
|
|
|
@@ -309,43 +360,54 @@ function impalaDagre(id) {
|
|
|
var max_node_time = 0;
|
|
|
plan["plan_nodes"].forEach(function(parent) {
|
|
|
max_node_time = Math.max(
|
|
|
- build(parent, null, edges, states, colour_idx, max_node_time));
|
|
|
+ build(parent, null, edges, states, colour_idx, max_node_time, 1, 1));
|
|
|
// Pick a new colour for each plan fragment
|
|
|
colour_idx = (colour_idx + 1) % colours.length;
|
|
|
});
|
|
|
-
|
|
|
+ var avgStates = average(states, 'max_time_val');
|
|
|
+ var edgesIO = edges.filter(function (edge) {
|
|
|
+ return edge.content.unit === 5;
|
|
|
+ });
|
|
|
+ var edgesNonIO = edges.filter(function (edge) {
|
|
|
+ return edge.content.unit === 0;
|
|
|
+ });
|
|
|
+ var avgEdgesIO = average(edgesIO, 'content.value');
|
|
|
+ var avgEdgesNonIO = average(edgesNonIO, 'content.value');
|
|
|
+ var avgCombined = averageCombined(avgStates, avgEdgesIO, states.length, edgesIO.length);
|
|
|
+ var avg = { '0': avgEdgesNonIO, '5': avgCombined};
|
|
|
// Keep a map of names to states for use when processing edges.
|
|
|
states.forEach(function(state) {
|
|
|
// Build the label for the node from the name and the detail
|
|
|
- var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + parseInt(state.name.split(':')[0], 10) + ");\">"; // TODO: Remove Hue dependency
|
|
|
- html += getIcon(state.icon)
|
|
|
- html += "<span class='name'>" + state.label + "</span><br/>";
|
|
|
- html += "<span class='metric'>" + state.max_time + "</span>";
|
|
|
+ var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + getId(state.name) + ");\">"; // TODO: Remove Hue dependency
|
|
|
+ html += getIcon(state.icon);
|
|
|
+ html += "<span style='display: inline-block;'><span class='name'>" + state.label + "</span><br/>";
|
|
|
+ var aboveAverageClass = state.max_time_val > avg ? 'above-average' : '';
|
|
|
+ html += "<span class='metric " + aboveAverageClass + "'>" + state.max_time + "</span>";
|
|
|
html += "<span class='detail'>" + state.detail + "</span><br/>";
|
|
|
- html += "<span class='id'>" + state.name + "</span>";
|
|
|
+ if (state.predicates) {
|
|
|
+ html += "<span class='detail'>" + state.predicates + "</span><br/>";
|
|
|
+ }
|
|
|
+ html += "<span class='id'>" + state.name + "</span></span>";
|
|
|
html += renderTimeline(state.name);
|
|
|
html += "</div>";
|
|
|
|
|
|
var style = state.style;
|
|
|
|
|
|
- // If colouring nodes by total time taken, choose a shade in the cols_by_time list
|
|
|
- // with idx proportional to the max time of the node divided by the max time over all
|
|
|
- // nodes.
|
|
|
- /*if (document.getElementById("colour_scheme").checked) {
|
|
|
- var idx = (cols_by_time.length - 1) * (state.max_time_val / (1.0 * max_node_time));
|
|
|
- style = "fill: " + cols_by_time[Math.floor(idx)];
|
|
|
- }*/
|
|
|
g.setNode(state.name, { "label": html,
|
|
|
"labelType": "html",
|
|
|
"style": style });
|
|
|
states_by_name[state.name] = state;
|
|
|
});
|
|
|
-
|
|
|
edges.forEach(function(edge) {
|
|
|
// Impala marks 'broadcast' as a property of the receiver, not the sender. We use
|
|
|
// '(BCAST)' to denote that a node is duplicating its output to all receivers.
|
|
|
- if (states_by_name[edge.end].is_broadcast) {
|
|
|
- edge.style.label += " * " + states_by_name[edge.end].num_instances;
|
|
|
+ /*if (states_by_name[edge.end].is_broadcast) {
|
|
|
+ if (states_by_name[edge.end].num_instances > 1) {
|
|
|
+ edge.style.label += " * " + states_by_name[edge.end].num_instances;
|
|
|
+ }
|
|
|
+ }*/
|
|
|
+ if (edge.content.value > avg[edge.content.unit]) {
|
|
|
+ edge.style.labelStyle = "font-weight: bold";
|
|
|
}
|
|
|
g.setEdge(edge.start, edge.end, edge.style);
|
|
|
});
|