Explorar el Código

HUE-8674 [jb] Add filters and groupby conditions to profile

jdesjean hace 6 años
padre
commit
e66d5bcd15

+ 6 - 1
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -186,6 +186,11 @@ class QueryApi(Api):
           return { 'font': 'fa-random' }
         else:
           return { 'font': 'fa-exchange' }
+      def get_sigma_icon (o):
+        if re.search(r'streaming', o['label_detail'], re.IGNORECASE):
+          return { 'svg': 'hi-sigma-stream' }
+        else:
+          return { 'svg': 'hi-sigma' }
       mapping = {
         'TOP-N': { 'type': 'TOPN', 'icon': { 'svg': 'hi-filter' } },
         'SORT': { 'type': 'SORT', 'icon': { 'svg': 'hi-sort' } },
@@ -195,7 +200,7 @@ class QueryApi(Api):
         'SCAN KUDU': { 'type': 'SCAN_KUDU', 'icon': { 'font': 'fa-table' } },
         'SCAN HBASE': { 'type': 'SCAN_HBASE', 'icon': { 'font': 'fa-th-large' } },
         'HASH JOIN': { 'type': 'HASH_JOIN', 'icon': { 'svg': 'hi-join' } },
-        'AGGREGATE': { 'type': 'AGGREGATE', 'icon': { 'svg': 'hi-sigma' } },
+        'AGGREGATE': { 'type': 'AGGREGATE', 'icon': { 'fn': get_sigma_icon } },
         'NESTED LOOP JOIN': { 'type': 'LOOP_JOIN', 'icon': { 'svg': 'hi-nested-loop' } },
         'SUBPLAN': { 'type': 'SUBPLAN', 'icon': { 'svg': 'hi-map' } },
         'UNNEST': { 'type': 'UNNEST', 'icon': { 'svg': 'hi-unnest' } },

La diferencia del archivo ha sido suprimido porque es demasiado grande
+ 0 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css


+ 21 - 9
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -113,17 +113,22 @@ function impalaDagre(id) {
     if (node["output_card"] === null || node["output_card"] === undefined) {
       return;
     }
+    var id = getId(node["label"]);
+    var metric_node = _impalaDagree._metrics && _impalaDagree._metrics.nodes[id]
+    var predicates = metric_node && (metric_node.other['group by'] || metric_node.other['hash predicates'] || metric_node.other['predicates']) || '';
     states.push({ "name": node["label"],
                   "type": node["type"],
                   "label": node["name"],
                   "detail": node["label_detail"],
+                  "predicates": predicates,
                   "num_instances": node["num_instances"],
                   "num_active": node["num_active"],
-                  "max_time": node["max_time"],
+                  "max_time": ko.bindingHandlers.numberFormat.human(node["max_time_val"], 5),
                   "avg_time": node["avg_time"],
                   "icon": node["icon"],
                   "is_broadcast": node["is_broadcast"],
-                  "max_time_val": node["max_time_val"]});
+                  "max_time_val": node["max_time_val"],
+                  "width": "200px"});
     var edgeCount;
     if (parent) {
       edgeCount = parseInt(node["output_card"], 10);
@@ -139,7 +144,7 @@ function impalaDagre(id) {
                    "val": edgeCount,
                    "style": { label: ko.bindingHandlers.simplesize.humanSize(edgeCount),
                               style: "stroke-dasharray: 5, 5;",
-                              labelpos: index === 0 ? 'l' : 'r' }});
+                              labelpos: 'l' }});
     }
     max_node_time = Math.max(node["max_time_val"], max_node_time)
     for (var i = 0; i < node["children"].length; ++i) {
@@ -165,13 +170,17 @@ function impalaDagre(id) {
     $("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
   }
 
+  function getId(name) {
+    return parseInt(name.split(':')[0], 10);
+  }
+
   function getKey(node) {
     var nodes = g.nodes();
     var key;
     var nNode = parseInt(node, 10);
     var keys = Object.keys(nodes);
     for (var i = 0; i < keys.length; i++) {
-      if (parseInt(nodes[keys[i]].split(':')[0], 10) == nNode) {
+      if (getId(nodes[keys[i]]) == nNode) {
         key = nodes[keys[i]];
         break;
       }
@@ -210,7 +219,7 @@ function impalaDagre(id) {
     if (!_impalaDagree._metrics) {
       return;
     }
-    var id = parseInt(key.split(':')[0], 10);
+    var id = getId(key);
     if (!_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[id].timeline) {
       return;
     }
@@ -330,14 +339,17 @@ function impalaDagre(id) {
     // Keep a map of names to states for use when processing edges.
     states.forEach(function(state) {
       // Build the label for the node from the name and the detail
-      var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + parseInt(state.name.split(':')[0], 10) + ");\">"; // TODO: Remove Hue dependency
-      html += getIcon(state.icon)
-      html += "<span class='name'>" + state.label + "</span><br/>";
+      var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + getId(state.name) + ");\">"; // TODO: Remove Hue dependency
+      html += getIcon(state.icon);
+      html += "<span style='display: inline-block;'><span class='name'>" + state.label + "</span><br/>";
       console.log(state.max_time_val + '-' + avg);
       var aboveAverageClass = state.max_time_val > avg ? 'above-average' : '';
       html += "<span class='metric " + aboveAverageClass + "'>" + state.max_time + "</span>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
-      html += "<span class='id'>" + state.name + "</span>";
+      if (state.predicates) {
+        html += "<span class='detail'>" + state.predicates + "</span><br/>";
+      }
+      html += "<span class='id'>" + state.name + "</span></span>";
       html += renderTimeline(state.name);
       html += "</div>";
 

+ 3 - 5
apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less

@@ -104,15 +104,13 @@
       right: 0px;
     }
     .name {
-      padding-right: 80px;
       text-transform: capitalize;
       font-size: 13px;
     }
     .detail {
       overflow: hidden;
       text-overflow: ellipsis;
-      width: calc(~"100% - 32px");
-      max-width: 185px;
+      width: 150px;
       display: inline-block;
       height: 14px;
       text-transform: lowercase;
@@ -121,11 +119,11 @@
       .fa {
         padding-top: 3px;
         padding-right: 2px;
-        float: left;
+        vertical-align: top;
       }
       .hi {
         padding-right: 5px;
-        float: left;
+        vertical-align: top;
       }
     }
     .fa {

+ 11 - 1
desktop/core/src/desktop/templates/hue_icons.mako

@@ -315,7 +315,17 @@
     <symbol id="hi-sigma" viewBox="0 0 32 32">
       <path d="M29.425 22.96l1.387-2.96h1.188l-2 12h-30v-2.32l10.361-12.225-10.361-10.361v-7.094h30.625l1.375 8h-1.074l-0.585-1.215c-1.104-2.293-1.934-2.785-4.341-2.785h-20.688l11.033 11.033-9.294 10.967h16.949c3.625 0 4.583-1.299 5.425-3.040z"></path>
     </symbol>
-
+    <symbol id="hi-sigma-stream" viewBox="0 0 32 32">
+      <g>
+        <path d="m24.29172,19.54425l1.13561,-2.50675l0.97268,0l-1.6375,10.1625l-24.5625,0l0,-1.96475l8.48307,-10.35304l-8.48307,-8.77447l0,-6.00773l25.07422,0l1.12578,6.775l-0.87933,0l-0.47897,-1.02896c-0.9039,-1.94189 -1.58346,-2.35854 -3.55419,-2.35854l-16.9383,0l9.03327,9.34357l-7.60946,9.28768l13.87699,0c2.96797,0 3.75233,-1.10009 4.44172,-2.5745l-0.00001,-0.00001z"/>
+      </g>
+      <g>
+       <path d="m30.15,17l-12.10001,0c-0.91094,0 -1.65,0.69107 -1.65,1.54285l0,11.31428c0,0.85179 0.73906,1.54285 1.65,1.54285l12.10001,0c0.91094,0 1.65,-0.69107 1.65,-1.54285l0,-11.31428c0,-0.85179 -0.73906,-1.54285 -1.65,-1.54285z" fill="white"/>
+      </g>
+      <g transform="translate(4 4) scale(0.05 0.05)">
+       <path d="m255.68435,333.84284l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714zm293.53124,50.53571l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714zm-39.1375,111.17856l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714z"/>
+      </g>
+    </symbol>
     <symbol id="hi-unnest" viewBox="0 0 32 32">
       <path d="m30,8l2,0l0,16l-2,0l0,-16z"/>
       <path d="m7,14l22,0l0,4l-22,0l0,5l-7,-7l7,-7l0,5z"/>

+ 6 - 1
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -28,6 +28,7 @@ from thrift.transport import TTransport
 from libanalyze import dot
 from libanalyze import gjson as jj
 from libanalyze import models
+from libanalyze import utils
 from libanalyze.rules import to_double
 
 
@@ -306,6 +307,8 @@ def summary(profile):
 
 def metrics(profile):
   execution_profile = profile.find_by_name('Execution Profile')
+  summary = profile.find_by_name("Summary")
+  plan_json = utils.parse_plan_details(summary.val.info_strings.get('Plan')) if summary.val.info_strings.get('Plan') else {}
   if not execution_profile:
     return {}
   counter_map = {'nodes': {}, 'max': 0}
@@ -322,7 +325,7 @@ def metrics(profile):
     host = node.augmented_host()
     metric_map = node.metric_map()
     if counter_map['nodes'].get(nid) is None:
-      counter_map['nodes'][nid] = {'properties': { 'hosts': {} }, 'children': { }, 'timeline': {'hosts': {}}}
+      counter_map['nodes'][nid] = {'properties': { 'hosts': {} }, 'children': { }, 'timeline': {'hosts': {}}, 'other': {}}
 
     event_list = node.event_list();
 
@@ -330,6 +333,8 @@ def metrics(profile):
       counter_map['nodes'][nid]['properties']['hosts'][host] = metric_map
       if event_list:
         counter_map['nodes'][nid]['timeline']['hosts'][host] = event_list
+      if plan_json.get(nid):
+        counter_map['nodes'][nid]['other'] = plan_json[nid]
     else:
       name = node.name()
       if counter_map['nodes'][nid]['children'].get(name) is None:

+ 4 - 3
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -589,12 +589,13 @@ class TopDownAnalysis:
         def add_host(node, exec_summary_json=exec_summary_json):
           is_plan_node = node.is_plan_node()
           node_id = node.id()
+          nid = int(node_id) if node_id and node.is_regular() else -1
            # Setup Hosts & Broadcast
-          if node_id and node.is_regular() and int(node_id) in exec_summary_json:
-            exec_summary_node = exec_summary_json.get(int(node_id), {})
+          if node_id and node.is_regular() and nid in exec_summary_json:
+            exec_summary_node = exec_summary_json.get(nid, {})
             node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node.get('hosts', ''), unit=0))
             broadcast = 0
-            if exec_summary_json[int(node_id)]["broadcast"]:
+            if exec_summary_json[nid]['broadcast']:
                 broadcast = 1
             node.val.counters.append(models.TCounter(name='Broadcast', value=broadcast, unit=0))
 

+ 8 - 2
desktop/libs/libanalyze/src/libanalyze/utils.py

@@ -77,9 +77,15 @@ def parse_exec_summary(summary_string):
 def parse_plan_details(plan_string):
     """Given a query plan, extracts the query details per node"""
     result = {}
+    last_id = -1
     for line in plan_string.split("\n"):
-        match = re.search(r'^(?!F)[|-]?(\d+):.*?\[(.*?)\]', line.strip())
+        match = re.search(r'(?!F)[|-]?(\d+):.*?\[(.*?)\]', line.strip())
         if match:
-            result[str(int(match.group(1)))] = match.group(2)
+          last_id = str(int(match.group(1)))
+          result[last_id] = {'detail': match.group(2)}
+        elif result.get(last_id):
+          match = re.search(r'[\|\s]*(.*?):\s?(.*)', line.strip())
+          if match:
+            result[last_id][match.group(1)] = match.group(2)
 
     return result

Algunos archivos no se mostraron porque demasiados archivos cambiaron en este cambio