Quellcode durchsuchen

HUE-8674 [jb] Add additional metrics to query profile.

jdesjean vor 7 Jahren
Ursprung
Commit
a61122c

+ 21 - 0
apps/impala/src/impala/api.py

@@ -158,6 +158,27 @@ def alanize(request):
     response['status'] = 0
   return JsonResponse(response)
 
+def alanize_metrics(request):
+  response = {'status': -1}
+  cluster = json.loads(request.POST.get('cluster', '{}'))
+  query_id = json.loads(request.POST.get('query_id'))
+
+  application = _get_server_name(cluster)
+  query_server = dbms.get_query_server_config()
+  session = Session.objects.get_session(request.user, query_server['server_name'])
+  server_url = _get_impala_server_url(session)
+
+  if query_id:
+    LOG.debug("Attempting to get Impala query profile at server_url %s for query ID: %s" % (server_url, query_id))
+    api = get_impalad_api(user=request.user, url=server_url)
+    query_profile = api.get_query_profile_encoded(query_id)
+    profile = analyzer.analyze(analyzer.parse_data(query_profile))
+    ANALYZER.pre_process(profile)
+    metrics = analyzer.metrics(profile)
+    response['data'] = { 'metrics': metrics }
+    response['status'] = 0
+  return JsonResponse(response)
+
 @require_POST
 @error_handler
 def alanize_fix(request):

+ 1 - 0
apps/impala/src/impala/urls.py

@@ -27,6 +27,7 @@ urlpatterns = [
   url(r'^api/query/(?P<query_history_id>\d+)/runtime_profile', impala_api.get_runtime_profile, name='get_runtime_profile'),
   url(r'^api/query/alanize$', impala_api.alanize, name='alanize'),
   url(r'^api/query/alanize/fix$', impala_api.alanize_fix, name='alanize_fix'),
+  url(r'^api/query/alanize/metrics', impala_api.alanize_metrics, name='alanize_metrics'),
 ]
 
 urlpatterns += beeswax_urls

+ 2 - 1
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -130,7 +130,8 @@ class QueryApi(Api):
         'profile': '',
         'plan': '',
         'backends': '',
-        'finstances': ''
+        'finstances': '',
+        'metrics': ''
       }
     })
 

Datei-Diff unterdrückt, da er zu groß ist
+ 0 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css


+ 71 - 32
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -12,19 +12,24 @@ function impalaDagre(id) {
   var g = new dagreD3.graphlib.Graph().setGraph({rankDir: "BT"});
   var svg = d3.select("#"+id + " svg");
   var inner = svg.select("g");
+  var states_by_name = { };
   var _impalaDagree = {
+    _metrics: {},
     init: function (initialScale) {
-      _impalaDagree.scale = initialScale;
+      clearSelection();
       zoom.translate([((svg.attr("width") || $("#"+id).width()) - g.graph().width * initialScale) / 2, 20])
       .scale(initialScale)
       .event(svg);
     },
+    metrics: function(metrics) {
+      _impalaDagree._metrics = metrics;
+    },
     update: function(plan) {
       renderGraph(plan);
       _impalaDagree._width = $(svg[0]).width();
     },
     height: function(value) {
-      var scale = _impalaDagree.scale || 1;
+      var scale = zoom.scale() || 1;
       var height = value || 600;
       _impalaDagree._height = height;
       svg.attr('height', height);
@@ -45,51 +50,46 @@ function impalaDagre(id) {
     },
     select: function(id) {
       select(id);
-    },
+    }
   };
   createActions();
 
   function createActions () {
+    svg.on('click', function () {
+      hideDetail();
+      clearSelection();
+    });
     d3.select("#"+id)
       .style('position', 'relative')
     .append('div')
-      .style('position', 'absolute')
-      .style('right', '5px')
-      .style('bottom', '5px')
       .classed('buttons', true)
-    .selectAll('button').data([{ type: 'reset', svg: 'hi-crop-free', divider: true }, { type: 'plus', icon: 'fa-plus', divider: true }, { type: 'minus', icon: 'fa-minus' }])
+    .selectAll('button').data([{ type: 'reset', svg: 'hi-crop-free', divider: true }, { type: 'plus', font: 'fa-plus', divider: true }, { type: 'minus', font: 'fa-minus' }])
     .enter()
     .append(function (data) {
       var text = "";
-      if (data.svg) {
-        text += "<div><svg class='hi'><use xlink:href='#"+ data.svg +"'></use></svg>";
-        if (data.divider) {
-          text += "<div class='divider'></div>";
-        }
-        text += "</div>";
-        button = $()[0];
-      } else if (data.icon) {
-        text += "<div><div class='fa fa-fw valign-middle " + data.icon + "'></div>";
-        if (data.divider) {
-          text += "<div class='divider'></div></div>";
-        }
-        text += "</div>";
+      text += '<div>';
+      text += getIcon(data);
+      if (data.divider) {
+        text += '<div class="divider"></div>';
       }
+      text += '</div>';
       var button = $(text)[0];
       $(button).on('click', function () {
         _impalaDagree.action(data.type);
       });
       return button;
     });
+    d3.select("#"+id)
+    .append('div')
+      .classed('details', true);
   }
 
   // Set up zoom support
   var zoom = d3.behavior.zoom().on("zoom", function() {
     var e = d3.event,
-        scale = Math.min(Math.max(e.scale, Math.min(_impalaDagree._width / g.graph().width, _impalaDagree._height / g.graph().height)), 2),
+        scale = Math.min(Math.max(e.scale, Math.min(Math.min(_impalaDagree._width / g.graph().width, _impalaDagree._height / g.graph().height), 1)), 2),
         tx = Math.min(40, Math.max(e.translate[0], _impalaDagree._width - 40 - g.graph().width * scale)),
         ty = Math.min(40, Math.max(e.translate[1], _impalaDagree._height - 40 - g.graph().height * scale));
-    _impalaDagree.scale = scale;
     zoom.translate([tx, ty]);
     zoom.scale(scale);
     inner.attr("transform", "translate(" + [tx, ty] + ")" +
@@ -150,8 +150,13 @@ function impalaDagre(id) {
     if (!key) {
       return;
     }
-    $("g.node").attr('class', 'node') // addClass doesn't work in svg on our version of jQuery
+    clearSelection();
     $("g.node:contains('" + key + "')").attr('class', 'node active');
+    showDetail(node);
+  }
+
+  function clearSelection() {
+    $("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
   }
 
   function getKey(node) {
@@ -185,6 +190,45 @@ function impalaDagre(id) {
             .scale(scale).event);
   }
 
+  function getIcon(icon) {
+    var html = '';
+    if (icon && icon.svg) {
+      html += '<svg class="hi"><use xlink:href="#'+ icon.svg +'"></use></svg>'
+    } else if (icon && icon.font) {
+      html += "<div class='fa fa-fw valign-middle " + icon.font + "'></div>";
+    }
+    return html;
+  }
+
+  function showDetail(id) {
+    var data;
+    if (_impalaDagree._metrics[id] && _impalaDagree._metrics[id]['averaged']) {
+      data = _impalaDagree._metrics[id]['averaged'];
+    } else if (_impalaDagree._metrics[id]) {
+      data = _impalaDagree._metrics[id][Object.keys(_impalaDagree._metrics[id])[0]];
+    }
+    d3.select('.query-plan').classed('open', true);
+    var title = d3.select('.query-plan .details')
+    .selectAll('.metric-title').data([0]);
+    title.enter().append('div').classed('metric-title', true);
+    var key = getKey(id);
+    title.html(getIcon(states_by_name[key].icon) + '<span>' + states_by_name[key].label+ '</span>');
+
+    var metricTitle = d3.select('.query-plan .details')
+    .selectAll('.metrics').data([0]);
+    metricTitle.enter().append('div').classed('metrics', true);
+
+    var metrics = d3.select('.query-plan .details .metrics').selectAll('div')
+    .data(Object.keys(data).sort().map(function (key) { return data[key]; }));
+    metrics.exit().remove();
+    metrics.enter().append('div');
+    metrics.html(function (datum) { return '<div class="metric-name">' + datum.name + '</div> : <div class="metric-value">' + ko.bindingHandlers.numberFormat.human(datum.value, datum.unit) + '</div>'; });
+  }
+
+  function hideDetail(id) {
+    d3.select('.query-plan').classed('open', false);
+  }
+
   function renderGraph(plan) {
     if (!plan || !plan.plan_nodes || !plan.plan_nodes.length) return;
     var states = [];
@@ -200,21 +244,16 @@ function impalaDagre(id) {
     });
 
     // Keep a map of names to states for use when processing edges.
-    var states_by_name = { };
     states.forEach(function(state) {
       // Build the label for the node from the name and the detail
-      var html = "";
-      if (state.icon && state.icon.svg) {
-        html += '<svg class="hi"><use xlink:href="#'+ state.icon.svg +'"></use></svg>'
-        //html += "<img src=\"" + icon.svg + "\"></img>";
-      } else if (state.icon && state.icon.font){
-        html += "<span class='fa fa-fw valign-middle " + state.icon.font + "'></span>";
-      }
+      var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + parseInt(state.name.split(':')[0], 10) + ");\">"; // TODO: Remove Hue dependency
+      html += getIcon(state.icon)
       html += "<span class='name'>" + state.label + "</span><br/>";
       html += "<span class='metric'>" + state.max_time + "</span>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
       html += "<span class='metric'>" + state.max_time + "</span>"
-      html += "<span class='id'>" + state.name + "</span>";;
+      html += "<span class='id'>" + state.name + "</span>";
+      html += "</div>";
 
       var style = state.style;
 

+ 75 - 10
apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less

@@ -91,6 +91,7 @@
 
   .query-plan {
     border: 1px solid @cui-gray-300;
+    overflow: hidden;
     .label,
     .badge {
       color: @cui-gray-800;
@@ -112,19 +113,25 @@
       width: calc(~"100% - 32px");
       display: inline-block;
     }
-    span.fa {
+    foreignObject {
+      .fa {
+        padding-top: 3px;
+        padding-right: 2px;
+        float: left;
+      }
+      .hi {
+        padding-right: 5px;
+        float: left;
+      }
+    }
+    .fa {
       color: @cui-gray-600;
-      float: left;
       font-size: 21px;
-      padding-top: 3px;
-      padding-right: 2px;
     }
-    svg .hi {
+    .hi {
       color: @cui-gray-600 !important;
-      float: left;
       width: 2.2em !important;
       height: 2.2em !important;
-      padding-right: 5px;
     }
     .buttons {
       .hi {
@@ -157,9 +164,11 @@
       }
     }
     .buttons {
+      position: absolute;
       background-color: white;
-      box-shadow: 0px 0px 2px 0px;
-      border: 1px solid @cui-gray-300;
+      box-shadow: 0px 0px 5px 0px;
+      right: 5px;
+      bottom: 5px;
       color: @cui-gray-600;
       div {
         line-height: 32px !important;
@@ -168,7 +177,7 @@
         height: 32px;
       }
     }
-    .button div:hover {
+    .buttons div:hover {
       color: @hue-primary-color-dark
     }
     .node rect {
@@ -189,6 +198,62 @@
     foreignObject > div {
       position: relative;
     }
+    .details {
+      background-color: white;
+      box-shadow: 0px 0px 10px 0px;
+      color: @cui-gray-600;
+      position: absolute;
+      display: none;
+      top: 0px;
+      right: 0px;
+      width: 200px;
+      height: 100%;
+      .metric-title {
+        background-color: @hue-primary-color-light;
+        font-weight: bold;
+        font-size: 16px;
+        color: @cui-gray-800;
+        line-height: 40px;
+        .fa {
+          padding-top: 0px;
+          line-height: 40px;
+          padding-right: 2px;
+        }
+        .hi {
+          padding-top: 2px;
+          padding-right: 5px;
+        }
+      }
+      .metrics {
+        overflow-y: scroll;
+        height: calc(~"100% - 40px");
+      }
+      .metric-name {
+        color: @cui-gray-800;
+        width: 120px;
+        overflow: hidden;
+        text-align: right;
+        display: inline-block;
+        white-space: nowrap;
+        vertical-align: middle;
+        font-weight: bold;
+        padding-left: 2px;
+      }
+      .metric-value {
+        color: @cui-gray-800;
+        overflow: hidden;
+        width: 50px;
+        vertical-align: middle;
+        display: inline-block;
+        white-space: nowrap;
+      }
+    }
+  }
+  .query-plan.open .details {
+    display: inherit;
+  }
+  .query-plan.open .buttons {
+    right: 208px;
   }
 
   div[data-jobType="queries"] pre {

+ 9 - 2
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -1496,7 +1496,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
     <div data-bind="css:{'span10': !$root.isMini(), 'span12 no-margin': $root.isMini() }">
       <ul class="nav nav-pills margin-top-20">
         <li>
-          <a href="#queries-page-plan${ SUFFIX }" data-bind="click: function(){ $('a[href=\'#queries-page-plan${ SUFFIX }\']').tab('show'); }, event: {'shown': function () { if (!properties.plan || !properties.plan().plan_json) { fetchProfile('plan'); } } }">
+          <a href="#queries-page-plan${ SUFFIX }" data-bind="click: function(){ $('a[href=\'#queries-page-plan${ SUFFIX }\']').tab('show'); }, event: {'shown': function () { if (!properties.plan || !properties.plan().plan_json) { fetchProfile('plan'); fetchMetrics(); } } }">
             ${ _('Plan') }</a>
         </li>
         <li>
@@ -1534,7 +1534,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       <div class="tab-content">
         <div class="tab-pane" id="queries-page-plan${ SUFFIX }" data-profile="plan">
           <div data-bind="visible:properties.plan && properties.plan().plan_json && properties.plan().plan_json.plan_nodes.length">
-            <div class="query-plan" id="queries-page-plan-graph${ SUFFIX }" data-bind="impalaDagre: {value: properties.plan && properties.plan().plan_json, height:$root.isMini() ? 535 : 600 }">
+            <div class="query-plan" id="queries-page-plan-graph${ SUFFIX }" data-bind="impalaDagre: {value: properties.plan && properties.plan().plan_json, metrics: properties.metrics && properties.metrics().metrics, height:$root.isMini() ? 535 : 600 }">
               <svg style="width:100%;height:100%;" id="queries-page-plan-svg${ SUFFIX }">
                 <defs>
                   <filter id="dropshadow" height="130%">
@@ -2709,6 +2709,13 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         });
       };
 
+      self.fetchMetrics = function (name, callback) {
+        ApiHelper.getInstance().fetchQueryExecutionStatistics({ queryId: self.id(), compute: '' })
+        .done(function(data) {
+          self.properties['metrics'](data);
+        });
+      };
+
       self.fetchStatus = function () {
         $.post("/jobbrowser/api/job", {
           cluster: ko.mapping.toJSON(vm.compute),

+ 23 - 0
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -2105,6 +2105,29 @@ var ApiHelper = (function () {
     return new CancellablePromise(deferred, request);
   };
 
+  ApiHelper.prototype.fetchQueryExecutionStatistics = function (options)  {
+    var self = this;
+    var url = '/impala/api/query/alanize/metrics';
+    var deferred = $.Deferred();
+
+    var request = self.simplePost(url, {
+      'cluster': JSON.stringify(options.compute),
+      'query_id': '"' + options.queryId + '"'
+      }, {
+      silenceErrors: options.silenceErrors,
+      successCallback: function (response) {
+        if (response.status === 0) {
+          deferred.resolve(response.data);
+        } else {
+          deferred.reject();
+        }
+      },
+      errorCallback: deferred.reject
+    });
+
+    return new CancellablePromise(deferred, request);
+  };
+
   /**
    * @param {Object} options
    * @param {boolean} [options.silenceErrors]

+ 1 - 0
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -7511,6 +7511,7 @@
       update: function (element, valueAccessor) {
         var props = ko.unwrap(valueAccessor());
         this._impalaDagre.update(props.value);
+        this._impalaDagre.metrics(props.metrics);
         this._impalaDagre.height(props.height);
       }
     };

+ 47 - 0
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -193,6 +193,27 @@ class Node(object):
         #frag_node = c
         return m.group(2)
 
+  def augmented_host(self):
+    if self.fragment_instance:
+      c = self.fragment_instance
+    elif self.fragment:
+      if self.fragment.is_averaged():
+        return 'averaged'
+      c = self.fragment.children[0]
+    elif self.is_fragment():
+      if self.is_averaged():
+        return 'averaged'
+      else:
+        c = self.children[0]
+    else:
+      return None
+    m = re.search(r'Instance\s(.*?)\s\(host=(.*?)\)', c.val.name)
+    if m:
+        #frag.instance_id = m.group(1)
+        #frag.host = m.group(2)
+        #frag_node = c
+        return m.group(2)
+
   def info_strings(self):
     return self.val.info_strings
 
@@ -209,6 +230,13 @@ class Node(object):
             ctr[c.name] = c
     return ctr
 
+  def metric_map(self):
+    ctr = {}
+    if self.val.counters:
+        for c in self.val.counters:
+            ctr[c.name] = { 'name': c.name, 'value': c.value, 'unit': c.unit }
+    return ctr
+
   def repr(self, indent):
     buffer = indent + self.val.name + "\n"
     if self.val.info_strings:
@@ -252,6 +280,25 @@ def summary(profile):
   peak_memory = models.TCounter(value=host_list[0][1], unit=3) if host_list else models.TCounter(value=0, unit=3) # The value is not always present
   return [{ 'key': 'PlanningTime', 'value': counter_map['PlanningTime'].value, 'unit': counter_map['PlanningTime'].unit }, {'key': 'RemoteFragmentsStarted', 'value': counter_map['RemoteFragmentsStarted'].value, 'unit': counter_map['RemoteFragmentsStarted'].unit}, {'key': 'TotalTime', 'value': counter_map_execution_profile['TotalTime'].value, 'unit': counter_map_execution_profile['TotalTime'].unit}, {'key': 'PeakMemoryUsage', 'value': peak_memory.value, 'unit': peak_memory.unit}]
 
+def metrics(profile):
+  execution_profile = profile.find_by_name('Execution Profile')
+  if not execution_profile:
+    return {}
+  counter_map = {}
+  def get_metric(node, counter_map=counter_map):
+    if not node.is_plan_node():
+      return
+    nid = node.id()
+    if counter_map.get(nid) is None:
+      counter_map[nid] = {}
+    host = node.augmented_host()
+    if host:
+      counter_map[nid][host] = node.metric_map()
+    else:
+      counter_map[nid] = node.metric_map()
+  execution_profile.foreach_lambda(get_metric)
+  return counter_map
+
 def heatmap_by_host(profile, counter_name):
   rows = models.host_by_metric(profile,
                                counter_name,

+ 4 - 4
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -533,7 +533,7 @@ class TopDownAnalysis:
 
     def pre_process(self, profile):
         summary = profile.find_by_name("Summary")
-        exec_summary_json = utils.parse_exec_summary(summary.val.info_strings['ExecSummary'])
+        exec_summary_json = utils.parse_exec_summary(summary.val.info_strings.get('ExecSummary')) if summary.val.info_strings.get('ExecSummary') else {}
         stats_mapping = {
           'Query Compilation': {
             'Metadata load finished': 'MetadataLoadTime',
@@ -592,14 +592,14 @@ class TopDownAnalysis:
           node_id = node.id()
            # Setup Hosts & Broadcast
           if node_id and node.is_regular() and int(node_id) in exec_summary_json:
-            exec_summary_node = exec_summary_json[int(node_id)]
-            node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node["hosts"], unit=0))
+            exec_summary_node = exec_summary_json.get(int(node_id), {})
+            node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node.get('hosts', ''), unit=0))
             broadcast = 0
             if exec_summary_json[int(node_id)]["broadcast"]:
                 broadcast = 1
             node.val.counters.append(models.TCounter(name='Broadcast', value=broadcast, unit=0))
 
-            if re.search(r'\w*_SCAN_NODE', node.name(), re.IGNORECASE):
+            if exec_summary_node.get('detail') and re.search(r'\w*_SCAN_NODE', node.name(), re.IGNORECASE):
               details = exec_summary_node['detail'].split()
               node.val.info_strings['Table'] = details[0]
               node.val.counters.append(models.TCounter(name='MissingStats', value=missing_stats.get(details[0], 0), unit=0))

Einige Dateien werden nicht angezeigt, da zu viele Dateien in diesem Diff geändert wurden.