Sfoglia il codice sorgente

HUE-8674 [jb] Add network time to profile.

jdesjean 6 anni fa
parent
commit
221a1679ed

+ 54 - 19
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -109,7 +109,7 @@ function impalaDagre(id) {
                       "#991F00", "#B22400", "#CC2900", "#E62E00", "#FF3300", "#FF4719"];
 
   // Recursively build a list of edges and states that comprise the plan graph
-  function build(node, parent, edges, states, colour_idx, max_node_time, index) {
+  function build(node, parent, edges, states, colour_idx, max_node_time, index, count) {
     if (node["output_card"] === null || node["output_card"] === undefined) {
       return;
     }
@@ -133,23 +133,25 @@ function impalaDagre(id) {
     if (parent) {
       edgeCount = parseInt(node["output_card"], 10);
       var label_val = "" + ko.bindingHandlers.simplesize.humanSize(edgeCount);
-      edges.push({ start: node["label"], end: parent, style: { label: label_val, labelpos: index === 0 ? 'l' : 'r' }, val: edgeCount });
+      edges.push({ start: node["label"], end: parent, style: { label: label_val, labelpos: index === 0 && count > 1 ? 'l' : 'r' }, content: { value: edgeCount, unit: 0 } });
     }
     // Add an inter-fragment edge. We use a red dashed line to show that rows are crossing
     // the fragment boundary.
     if (node["data_stream_target"]) {
       edgeCount = parseInt(node["output_card"], 10);
+      var sendTime = getMaxTotalNetworkSendTime(node["label"]);
+      var text = sendTime && ko.bindingHandlers.numberFormat.human(sendTime.value, sendTime.unit) || ko.bindingHandlers.simplesize.humanSize(edgeCount);
       edges.push({ "start": node["label"],
                    "end": node["data_stream_target"],
-                   "val": edgeCount,
-                   "style": { label: ko.bindingHandlers.simplesize.humanSize(edgeCount),
+                   "content": sendTime ? sendTime : { value: edgeCount, unit: 0 },
+                   "style": { label: text,
                               style: "stroke-dasharray: 5, 5;",
-                              labelpos: 'l' }});
+                              labelpos: index === 0 && count > 1 ? 'l' : 'r' }});
     }
     max_node_time = Math.max(node["max_time_val"], max_node_time)
     for (var i = 0; i < node["children"].length; ++i) {
       max_node_time = build(
-        node["children"][i], node["label"], edges, states, colour_idx, max_node_time, i);
+        node["children"][i], node["label"], edges, states, colour_idx, max_node_time, i, node["children"].length);
     }
     return max_node_time;
   }
@@ -170,8 +172,8 @@ function impalaDagre(id) {
     $("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
   }
 
-  function getId(name) {
-    return parseInt(name.split(':')[0], 10);
+  function getId(key) {
+    return parseInt(key.split(':')[0], 10);
   }
 
   function getKey(node) {
@@ -215,6 +217,21 @@ function impalaDagre(id) {
     return html;
   }
 
+  function getMaxTotalNetworkSendTime(node) {
+    var id = getId(node);
+    if (!_impalaDagree._metrics || !_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[_impalaDagree._metrics.nodes[id].fragment]) {
+      return;
+    }
+    var fragment = _impalaDagree._metrics.nodes[_impalaDagree._metrics.nodes[id].fragment];
+    return Object.keys(fragment.properties.hosts).reduce(function (previous, host) {
+      if (fragment.properties.hosts[host].TotalNetworkSendTime.value > previous.value) {
+        return fragment.properties.hosts[host].TotalNetworkSendTime;
+      } else {
+        return previous;
+      }
+    }, { value: -1, unit: 5 });
+  }
+
   function getTimelineData(key) {
     if (!_impalaDagree._metrics) {
       return;
@@ -313,12 +330,24 @@ function impalaDagre(id) {
     d3.select('.query-plan').classed('open', false);
   }
 
+  function getProperty(object, path) {
+    var keys = path.split('.');
+    for (var i = 0; i < keys.length; i++) {
+      object = object[keys[i]];
+    }
+    return object;
+  }
+
   function average(states, metric) {
     var sum = 0;
     for (var i = 0; i < states.length; i++) {
-      sum += states[i][metric];
+      sum += getProperty(states[i], metric);
     }
-    return sum / states.length;
+    return states.length > 0 ? sum / states.length : 0;
+  }
+
+  function averageCombined(avg1, avg2, count1, count2) {
+    return (avg1 * count1 + avg2 * count2) / (count1 + count2);
   }
 
   function renderGraph() {
@@ -331,18 +360,27 @@ function impalaDagre(id) {
     var max_node_time = 0;
     plan["plan_nodes"].forEach(function(parent) {
       max_node_time = Math.max(
-        build(parent, null, edges, states, colour_idx, max_node_time));
+        build(parent, null, edges, states, colour_idx, max_node_time, 1, 1));
       // Pick a new colour for each plan fragment
       colour_idx = (colour_idx + 1) % colours.length;
     });
-    var avg = average(states, 'max_time_val');
+    var avgStates = average(states, 'max_time_val');
+    var edgesIO = edges.filter(function (edge) {
+      return edge.content.unit === 5;
+    });
+    var edgesNonIO = edges.filter(function (edge) {
+      return edge.content.unit === 0;
+    });
+    var avgEdgesIO = average(edgesIO, 'content.value');
+    var avgEdgesNonIO = average(edgesNonIO, 'content.value');
+    var avgCombined = averageCombined(avgStates, avgEdgesIO, states.length, edgesIO.length);
+    var avg = { '0': avgEdgesNonIO, '5': avgCombined};
     // Keep a map of names to states for use when processing edges.
     states.forEach(function(state) {
       // Build the label for the node from the name and the detail
       var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + getId(state.name) + ");\">"; // TODO: Remove Hue dependency
       html += getIcon(state.icon);
       html += "<span style='display: inline-block;'><span class='name'>" + state.label + "</span><br/>";
-      console.log(state.max_time_val + '-' + avg);
       var aboveAverageClass = state.max_time_val > avg ? 'above-average' : '';
       html += "<span class='metric " + aboveAverageClass + "'>" + state.max_time + "</span>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
@@ -360,18 +398,15 @@ function impalaDagre(id) {
                               "style": style });
       states_by_name[state.name] = state;
     });
-
-    var avgEdges = average(edges, 'val');
-
     edges.forEach(function(edge) {
       // Impala marks 'broadcast' as a property of the receiver, not the sender. We use
       // '(BCAST)' to denote that a node is duplicating its output to all receivers.
-      if (states_by_name[edge.end].is_broadcast) {
+      /*if (states_by_name[edge.end].is_broadcast) {
         if (states_by_name[edge.end].num_instances > 1) {
           edge.style.label += " * " + states_by_name[edge.end].num_instances;
         }
-      }
-      if (edge.val > avgEdges) {
+      }*/
+      if (edge.content.value > avg[edge.content.unit]) {
         edge.style.labelStyle = "font-weight: bold";
       }
       g.setEdge(edge.start, edge.end, edge.style);

+ 2 - 2
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -7577,10 +7577,10 @@
         value = value * 1;
         if (value < Math.pow(10, 3)) {
           return value + " ns";
-        } else if (value < Math.pow(10, 6)) {
+        } else if (value - Math.pow(10, 6) < -Math.pow(10, 3) / 2) { // Make sure rounding doesn't cause numbers to have more than 4 significant digits.
           value = (value * 1.0) / Math.pow(10, 3);
           return sprintf("%.1f us", value);
-        } else if (value < Math.pow(10, 9)) {
+        } else if (value - Math.pow(10, 9) < -Math.pow(10, 6) / 2) {
           value = (value * 1.0) / Math.pow(10, 6);
           return sprintf("%.1f ms", value);
         } else {

+ 16 - 7
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -81,8 +81,8 @@ class Node(object):
 
   def is_regular(self):
     id = self.id()
-    matches = id and re.search(r'[a-zA-Z]+', id)
-    return id and matches is None
+    matches = id and re.search(r'^\d*$', id)
+    return id and matches
 
   def name(self):
     matches = re.search(r'(.*?)(\s+\(((dst_)?id)=(\d+)\))?$', self.val.name)
@@ -314,9 +314,15 @@ def metrics(profile):
   counter_map = {'nodes': {}, 'max': 0}
   def flatten(node, counter_map=counter_map):
     is_plan_node = node.is_plan_node()
+    is_parent_node = is_plan_node
     if not is_plan_node:
       if node.plan_node:
         nid = node.plan_node.id()
+      elif node.is_fragment_instance():
+        is_parent_node = True
+        nid = node.fragment.id()
+      elif node.fragment:
+        nid = node.fragment.id()
       else:
         return
     else:
@@ -329,12 +335,14 @@ def metrics(profile):
 
     event_list = node.event_list();
 
-    if is_plan_node:
+    if is_parent_node:
       counter_map['nodes'][nid]['properties']['hosts'][host] = metric_map
       if event_list:
         counter_map['nodes'][nid]['timeline']['hosts'][host] = event_list
       if plan_json.get(nid):
         counter_map['nodes'][nid]['other'] = plan_json[nid]
+      if is_plan_node:
+        counter_map['nodes'][nid]['fragment'] = node.fragment.id()
     else:
       name = node.name()
       if counter_map['nodes'][nid]['children'].get(name) is None:
@@ -346,10 +354,11 @@ def metrics(profile):
   for nodeid, node in counter_map['nodes'].iteritems():
     host_min = {'value': sys.maxint, 'host' : None}
     for host_name, host_value in node['timeline']['hosts'].iteritems():
-      value = host_value['Node Lifecycle Event Timeline'][len(host_value['Node Lifecycle Event Timeline']) - 1]['value']
-      if value < host_min['value']:
-        host_min['value'] = value
-        host_min['host'] = host_name
+      if host_value.get('Node Lifecycle Event Timeline'):
+        value = host_value['Node Lifecycle Event Timeline'][len(host_value['Node Lifecycle Event Timeline']) - 1]['value']
+        if value < host_min['value']:
+          host_min['value'] = value
+          host_min['host'] = host_name
     node['timeline']['min'] = host_min.get('host', '')
     if node['timeline']['min']:
       node_min = node['timeline']['hosts'][node['timeline']['min']]['Node Lifecycle Event Timeline']