Browse Source

HUE-8674 [jb] Add filters and groupby conditions to profile

jdesjean 6 years ago
parent
commit
e66d5bc

+ 6 - 1
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -186,6 +186,11 @@ class QueryApi(Api):
           return { 'font': 'fa-random' }
           return { 'font': 'fa-random' }
         else:
         else:
           return { 'font': 'fa-exchange' }
           return { 'font': 'fa-exchange' }
+      def get_sigma_icon (o):
+        if re.search(r'streaming', o['label_detail'], re.IGNORECASE):
+          return { 'svg': 'hi-sigma-stream' }
+        else:
+          return { 'svg': 'hi-sigma' }
       mapping = {
       mapping = {
         'TOP-N': { 'type': 'TOPN', 'icon': { 'svg': 'hi-filter' } },
         'TOP-N': { 'type': 'TOPN', 'icon': { 'svg': 'hi-filter' } },
         'SORT': { 'type': 'SORT', 'icon': { 'svg': 'hi-sort' } },
         'SORT': { 'type': 'SORT', 'icon': { 'svg': 'hi-sort' } },
@@ -195,7 +200,7 @@ class QueryApi(Api):
         'SCAN KUDU': { 'type': 'SCAN_KUDU', 'icon': { 'font': 'fa-table' } },
         'SCAN KUDU': { 'type': 'SCAN_KUDU', 'icon': { 'font': 'fa-table' } },
         'SCAN HBASE': { 'type': 'SCAN_HBASE', 'icon': { 'font': 'fa-th-large' } },
         'SCAN HBASE': { 'type': 'SCAN_HBASE', 'icon': { 'font': 'fa-th-large' } },
         'HASH JOIN': { 'type': 'HASH_JOIN', 'icon': { 'svg': 'hi-join' } },
         'HASH JOIN': { 'type': 'HASH_JOIN', 'icon': { 'svg': 'hi-join' } },
-        'AGGREGATE': { 'type': 'AGGREGATE', 'icon': { 'svg': 'hi-sigma' } },
+        'AGGREGATE': { 'type': 'AGGREGATE', 'icon': { 'fn': get_sigma_icon } },
         'NESTED LOOP JOIN': { 'type': 'LOOP_JOIN', 'icon': { 'svg': 'hi-nested-loop' } },
         'NESTED LOOP JOIN': { 'type': 'LOOP_JOIN', 'icon': { 'svg': 'hi-nested-loop' } },
         'SUBPLAN': { 'type': 'SUBPLAN', 'icon': { 'svg': 'hi-map' } },
         'SUBPLAN': { 'type': 'SUBPLAN', 'icon': { 'svg': 'hi-map' } },
         'UNNEST': { 'type': 'UNNEST', 'icon': { 'svg': 'hi-unnest' } },
         'UNNEST': { 'type': 'UNNEST', 'icon': { 'svg': 'hi-unnest' } },

File diff suppressed because it is too large
+ 0 - 0
apps/jobbrowser/src/jobbrowser/static/jobbrowser/css/jobbrowser-embeddable.css


+ 21 - 9
apps/jobbrowser/src/jobbrowser/static/jobbrowser/js/impala_dagre.js

@@ -113,17 +113,22 @@ function impalaDagre(id) {
     if (node["output_card"] === null || node["output_card"] === undefined) {
     if (node["output_card"] === null || node["output_card"] === undefined) {
       return;
       return;
     }
     }
+    var id = getId(node["label"]);
+    var metric_node = _impalaDagree._metrics && _impalaDagree._metrics.nodes[id]
+    var predicates = metric_node && (metric_node.other['group by'] || metric_node.other['hash predicates'] || metric_node.other['predicates']) || '';
     states.push({ "name": node["label"],
     states.push({ "name": node["label"],
                   "type": node["type"],
                   "type": node["type"],
                   "label": node["name"],
                   "label": node["name"],
                   "detail": node["label_detail"],
                   "detail": node["label_detail"],
+                  "predicates": predicates,
                   "num_instances": node["num_instances"],
                   "num_instances": node["num_instances"],
                   "num_active": node["num_active"],
                   "num_active": node["num_active"],
-                  "max_time": node["max_time"],
+                  "max_time": ko.bindingHandlers.numberFormat.human(node["max_time_val"], 5),
                   "avg_time": node["avg_time"],
                   "avg_time": node["avg_time"],
                   "icon": node["icon"],
                   "icon": node["icon"],
                   "is_broadcast": node["is_broadcast"],
                   "is_broadcast": node["is_broadcast"],
-                  "max_time_val": node["max_time_val"]});
+                  "max_time_val": node["max_time_val"],
+                  "width": "200px"});
     var edgeCount;
     var edgeCount;
     if (parent) {
     if (parent) {
       edgeCount = parseInt(node["output_card"], 10);
       edgeCount = parseInt(node["output_card"], 10);
@@ -139,7 +144,7 @@ function impalaDagre(id) {
                    "val": edgeCount,
                    "val": edgeCount,
                    "style": { label: ko.bindingHandlers.simplesize.humanSize(edgeCount),
                    "style": { label: ko.bindingHandlers.simplesize.humanSize(edgeCount),
                               style: "stroke-dasharray: 5, 5;",
                               style: "stroke-dasharray: 5, 5;",
-                              labelpos: index === 0 ? 'l' : 'r' }});
+                              labelpos: 'l' }});
     }
     }
     max_node_time = Math.max(node["max_time_val"], max_node_time)
     max_node_time = Math.max(node["max_time_val"], max_node_time)
     for (var i = 0; i < node["children"].length; ++i) {
     for (var i = 0; i < node["children"].length; ++i) {
@@ -165,13 +170,17 @@ function impalaDagre(id) {
     $("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
     $("g.node").attr('class', 'node'); // addClass doesn't work in svg on our version of jQuery
   }
   }
 
 
+  function getId(name) {
+    return parseInt(name.split(':')[0], 10);
+  }
+
   function getKey(node) {
   function getKey(node) {
     var nodes = g.nodes();
     var nodes = g.nodes();
     var key;
     var key;
     var nNode = parseInt(node, 10);
     var nNode = parseInt(node, 10);
     var keys = Object.keys(nodes);
     var keys = Object.keys(nodes);
     for (var i = 0; i < keys.length; i++) {
     for (var i = 0; i < keys.length; i++) {
-      if (parseInt(nodes[keys[i]].split(':')[0], 10) == nNode) {
+      if (getId(nodes[keys[i]]) == nNode) {
         key = nodes[keys[i]];
         key = nodes[keys[i]];
         break;
         break;
       }
       }
@@ -210,7 +219,7 @@ function impalaDagre(id) {
     if (!_impalaDagree._metrics) {
     if (!_impalaDagree._metrics) {
       return;
       return;
     }
     }
-    var id = parseInt(key.split(':')[0], 10);
+    var id = getId(key);
     if (!_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[id].timeline) {
     if (!_impalaDagree._metrics.nodes[id] || !_impalaDagree._metrics.nodes[id].timeline) {
       return;
       return;
     }
     }
@@ -330,14 +339,17 @@ function impalaDagre(id) {
     // Keep a map of names to states for use when processing edges.
     // Keep a map of names to states for use when processing edges.
     states.forEach(function(state) {
     states.forEach(function(state) {
       // Build the label for the node from the name and the detail
       // Build the label for the node from the name and the detail
-      var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + parseInt(state.name.split(':')[0], 10) + ");\">"; // TODO: Remove Hue dependency
-      html += getIcon(state.icon)
-      html += "<span class='name'>" + state.label + "</span><br/>";
+      var html = "<div onclick=\"event.stopPropagation(); huePubSub.publish('impala.node.select', " + getId(state.name) + ");\">"; // TODO: Remove Hue dependency
+      html += getIcon(state.icon);
+      html += "<span style='display: inline-block;'><span class='name'>" + state.label + "</span><br/>";
       console.log(state.max_time_val + '-' + avg);
       console.log(state.max_time_val + '-' + avg);
       var aboveAverageClass = state.max_time_val > avg ? 'above-average' : '';
       var aboveAverageClass = state.max_time_val > avg ? 'above-average' : '';
       html += "<span class='metric " + aboveAverageClass + "'>" + state.max_time + "</span>";
       html += "<span class='metric " + aboveAverageClass + "'>" + state.max_time + "</span>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
       html += "<span class='detail'>" + state.detail + "</span><br/>";
-      html += "<span class='id'>" + state.name + "</span>";
+      if (state.predicates) {
+        html += "<span class='detail'>" + state.predicates + "</span><br/>";
+      }
+      html += "<span class='id'>" + state.name + "</span></span>";
       html += renderTimeline(state.name);
       html += renderTimeline(state.name);
       html += "</div>";
       html += "</div>";
 
 

+ 3 - 5
apps/jobbrowser/src/jobbrowser/static/jobbrowser/less/jobbrowser-embeddable.less

@@ -104,15 +104,13 @@
       right: 0px;
       right: 0px;
     }
     }
     .name {
     .name {
-      padding-right: 80px;
       text-transform: capitalize;
       text-transform: capitalize;
       font-size: 13px;
       font-size: 13px;
     }
     }
     .detail {
     .detail {
       overflow: hidden;
       overflow: hidden;
       text-overflow: ellipsis;
       text-overflow: ellipsis;
-      width: calc(~"100% - 32px");
-      max-width: 185px;
+      width: 150px;
       display: inline-block;
       display: inline-block;
       height: 14px;
       height: 14px;
       text-transform: lowercase;
       text-transform: lowercase;
@@ -121,11 +119,11 @@
       .fa {
       .fa {
         padding-top: 3px;
         padding-top: 3px;
         padding-right: 2px;
         padding-right: 2px;
-        float: left;
+        vertical-align: top;
       }
       }
       .hi {
       .hi {
         padding-right: 5px;
         padding-right: 5px;
-        float: left;
+        vertical-align: top;
       }
       }
     }
     }
     .fa {
     .fa {

+ 11 - 1
desktop/core/src/desktop/templates/hue_icons.mako

@@ -315,7 +315,17 @@
     <symbol id="hi-sigma" viewBox="0 0 32 32">
     <symbol id="hi-sigma" viewBox="0 0 32 32">
       <path d="M29.425 22.96l1.387-2.96h1.188l-2 12h-30v-2.32l10.361-12.225-10.361-10.361v-7.094h30.625l1.375 8h-1.074l-0.585-1.215c-1.104-2.293-1.934-2.785-4.341-2.785h-20.688l11.033 11.033-9.294 10.967h16.949c3.625 0 4.583-1.299 5.425-3.040z"></path>
       <path d="M29.425 22.96l1.387-2.96h1.188l-2 12h-30v-2.32l10.361-12.225-10.361-10.361v-7.094h30.625l1.375 8h-1.074l-0.585-1.215c-1.104-2.293-1.934-2.785-4.341-2.785h-20.688l11.033 11.033-9.294 10.967h16.949c3.625 0 4.583-1.299 5.425-3.040z"></path>
     </symbol>
     </symbol>
-
+    <symbol id="hi-sigma-stream" viewBox="0 0 32 32">
+      <g>
+        <path d="m24.29172,19.54425l1.13561,-2.50675l0.97268,0l-1.6375,10.1625l-24.5625,0l0,-1.96475l8.48307,-10.35304l-8.48307,-8.77447l0,-6.00773l25.07422,0l1.12578,6.775l-0.87933,0l-0.47897,-1.02896c-0.9039,-1.94189 -1.58346,-2.35854 -3.55419,-2.35854l-16.9383,0l9.03327,9.34357l-7.60946,9.28768l13.87699,0c2.96797,0 3.75233,-1.10009 4.44172,-2.5745l-0.00001,-0.00001z"/>
+      </g>
+      <g>
+       <path d="m30.15,17l-12.10001,0c-0.91094,0 -1.65,0.69107 -1.65,1.54285l0,11.31428c0,0.85179 0.73906,1.54285 1.65,1.54285l12.10001,0c0.91094,0 1.65,-0.69107 1.65,-1.54285l0,-11.31428c0,-0.85179 -0.73906,-1.54285 -1.65,-1.54285z" fill="white"/>
+      </g>
+      <g transform="translate(4 4) scale(0.05 0.05)">
+       <path d="m255.68435,333.84284l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714zm293.53124,50.53571l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714zm-39.1375,111.17856l-254.39374,0c-5.40587,0 -9.78438,4.52295 -9.78438,10.10714l0,40.42857c0,5.5842 4.37851,10.10714 9.78438,10.10714l254.39374,0c5.40587,0 9.78438,-4.52295 9.78438,-10.10714l0,-40.42857c0,-5.5842 -4.37851,-10.10714 -9.78438,-10.10714z"/>
+      </g>
+    </symbol>
     <symbol id="hi-unnest" viewBox="0 0 32 32">
     <symbol id="hi-unnest" viewBox="0 0 32 32">
       <path d="m30,8l2,0l0,16l-2,0l0,-16z"/>
       <path d="m30,8l2,0l0,16l-2,0l0,-16z"/>
       <path d="m7,14l22,0l0,4l-22,0l0,5l-7,-7l7,-7l0,5z"/>
       <path d="m7,14l22,0l0,4l-22,0l0,5l-7,-7l7,-7l0,5z"/>

+ 6 - 1
desktop/libs/libanalyze/src/libanalyze/analyze.py

@@ -28,6 +28,7 @@ from thrift.transport import TTransport
 from libanalyze import dot
 from libanalyze import dot
 from libanalyze import gjson as jj
 from libanalyze import gjson as jj
 from libanalyze import models
 from libanalyze import models
+from libanalyze import utils
 from libanalyze.rules import to_double
 from libanalyze.rules import to_double
 
 
 
 
@@ -306,6 +307,8 @@ def summary(profile):
 
 
 def metrics(profile):
 def metrics(profile):
   execution_profile = profile.find_by_name('Execution Profile')
   execution_profile = profile.find_by_name('Execution Profile')
+  summary = profile.find_by_name("Summary")
+  plan_json = utils.parse_plan_details(summary.val.info_strings.get('Plan')) if summary.val.info_strings.get('Plan') else {}
   if not execution_profile:
   if not execution_profile:
     return {}
     return {}
   counter_map = {'nodes': {}, 'max': 0}
   counter_map = {'nodes': {}, 'max': 0}
@@ -322,7 +325,7 @@ def metrics(profile):
     host = node.augmented_host()
     host = node.augmented_host()
     metric_map = node.metric_map()
     metric_map = node.metric_map()
     if counter_map['nodes'].get(nid) is None:
     if counter_map['nodes'].get(nid) is None:
-      counter_map['nodes'][nid] = {'properties': { 'hosts': {} }, 'children': { }, 'timeline': {'hosts': {}}}
+      counter_map['nodes'][nid] = {'properties': { 'hosts': {} }, 'children': { }, 'timeline': {'hosts': {}}, 'other': {}}
 
 
     event_list = node.event_list();
     event_list = node.event_list();
 
 
@@ -330,6 +333,8 @@ def metrics(profile):
       counter_map['nodes'][nid]['properties']['hosts'][host] = metric_map
       counter_map['nodes'][nid]['properties']['hosts'][host] = metric_map
       if event_list:
       if event_list:
         counter_map['nodes'][nid]['timeline']['hosts'][host] = event_list
         counter_map['nodes'][nid]['timeline']['hosts'][host] = event_list
+      if plan_json.get(nid):
+        counter_map['nodes'][nid]['other'] = plan_json[nid]
     else:
     else:
       name = node.name()
       name = node.name()
       if counter_map['nodes'][nid]['children'].get(name) is None:
       if counter_map['nodes'][nid]['children'].get(name) is None:

+ 4 - 3
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -589,12 +589,13 @@ class TopDownAnalysis:
         def add_host(node, exec_summary_json=exec_summary_json):
         def add_host(node, exec_summary_json=exec_summary_json):
           is_plan_node = node.is_plan_node()
           is_plan_node = node.is_plan_node()
           node_id = node.id()
           node_id = node.id()
+          nid = int(node_id) if node_id and node.is_regular() else -1
            # Setup Hosts & Broadcast
            # Setup Hosts & Broadcast
-          if node_id and node.is_regular() and int(node_id) in exec_summary_json:
-            exec_summary_node = exec_summary_json.get(int(node_id), {})
+          if node_id and node.is_regular() and nid in exec_summary_json:
+            exec_summary_node = exec_summary_json.get(nid, {})
             node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node.get('hosts', ''), unit=0))
             node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node.get('hosts', ''), unit=0))
             broadcast = 0
             broadcast = 0
-            if exec_summary_json[int(node_id)]["broadcast"]:
+            if exec_summary_json[nid]['broadcast']:
                 broadcast = 1
                 broadcast = 1
             node.val.counters.append(models.TCounter(name='Broadcast', value=broadcast, unit=0))
             node.val.counters.append(models.TCounter(name='Broadcast', value=broadcast, unit=0))
 
 

+ 8 - 2
desktop/libs/libanalyze/src/libanalyze/utils.py

@@ -77,9 +77,15 @@ def parse_exec_summary(summary_string):
 def parse_plan_details(plan_string):
 def parse_plan_details(plan_string):
     """Given a query plan, extracts the query details per node"""
     """Given a query plan, extracts the query details per node"""
     result = {}
     result = {}
+    last_id = -1
     for line in plan_string.split("\n"):
     for line in plan_string.split("\n"):
-        match = re.search(r'^(?!F)[|-]?(\d+):.*?\[(.*?)\]', line.strip())
+        match = re.search(r'(?!F)[|-]?(\d+):.*?\[(.*?)\]', line.strip())
         if match:
         if match:
-            result[str(int(match.group(1)))] = match.group(2)
+          last_id = str(int(match.group(1)))
+          result[last_id] = {'detail': match.group(2)}
+        elif result.get(last_id):
+          match = re.search(r'[\|\s]*(.*?):\s?(.*)', line.strip())
+          if match:
+            result[last_id][match.group(1)] = match.group(2)
 
 
     return result
     return result

Some files were not shown because too many files changed in this diff