浏览代码

HUE-8665 [editor] Add rule to check if statistics are missing.

(cherry picked from commit 79d8a7ae2281487b4ae38bc23dbe0693b33bda4e)
jdesjean 7 年之前
父节点
当前提交
641f23a

+ 3 - 1
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -7526,8 +7526,10 @@
         var fn;
         if (unwrapped.unit == 3) {
           fn = ko.bindingHandlers.bytesize.humanSize
-        } else { // 5
+        } else if (unwrapped.unit == 5) {
           fn = ko.bindingHandlers.duration.humanTime
+        } else {
+          fn = function(value){ return value; }
         }
         var formatted = fn(unwrapped.value);
         $(element).text(formatted);

+ 1 - 1
desktop/core/src/desktop/templates/ko_components/ko_execution_analysis.mako

@@ -62,7 +62,7 @@ from desktop.views import _ko
               <div><span data-bind="text: contribution_factor_str"></span> - <strong><span data-bind="duration: wall_clock_time"></strong></div>
               <ol data-bind="foreach: reason">
                 <li>
-                  <span data-bind="text: message"></span><strong> - <span data-bind="duration: impact"></span></strong>
+                  <span data-bind="text: message"></span><strong> - <span data-bind="numberFormat: { value: impact, unit: unit }"></span></strong>
                 </li>
               </ol>
             </li>

+ 2 - 2
desktop/libs/libanalyze/reasons/scanner_parallelism.json

@@ -5,10 +5,10 @@
     "AverageScannerThreadConcurrency",
     "LocalTime"
   ],
-  "unit_id": 0,
+  "unit_id": 5,
   "rule": {
     "condition": "vars['IOBound']==False",
-    "expr": "(8 - vars['AverageScannerThreadConcurrency']) / 8 * vars['LocalTime']",
+    "expr": "float(8 - vars['AverageScannerThreadConcurrency']) / 8.0 * vars['LocalTime']",
     "message": "Speed can be improved by that much if there's 8 scanner threads",
     "label": "Lack of scanner thread parallelism",
     "prio": 2

+ 13 - 0
desktop/libs/libanalyze/reasons/stats_missing.json

@@ -0,0 +1,13 @@
+{
+  "type": "SQLOperator",
+  "node_name": "HDFS_SCAN_NODE",
+  "metric_names": [
+    "MissingStats"
+  ],
+  "rule": {
+    "condition": "vars['MissingStats'] == 1",
+    "expr": "1",
+    "message": "The statistics are missing or corrupt which prevent scan and join optimizations.",
+    "label": "Statistics Missing"
+  }
+}

+ 1 - 0
desktop/libs/libanalyze/src/libanalyze/models.py

@@ -35,6 +35,7 @@ class Reason(object):
   def __init__(self, **kwargs):
     self.message = None
     self.impact = None
+    self.unit = None
     self.__dict__.update(kwargs)
 
   def to_json(self):

+ 25 - 9
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -161,7 +161,8 @@ class SQLOperatorReason:
         msg = self.rule["label"] + ": " + self.rule["message"]
         return {
             "impact": impact,
-            "message": msg
+            "message": msg,
+            "unit": self.kwargs.get('unit_id', -1)
         }
 
     def check_exprs(self, group):
@@ -228,7 +229,8 @@ class SummaryReason(SQLOperatorReason):
         msg = self.rule["label"] + ": " + self.rule["message"]
         return {
             "impact": impact,
-            "message": msg
+            "message": msg,
+            "unit": self.kwargs.get('unit_id', -1)
         }
 
 class JoinOrderStrategyCheck(SQLOperatorReason):
@@ -276,7 +278,8 @@ class JoinOrderStrategyCheck(SQLOperatorReason):
         impact = (networkcost - min(bcost, scost) - 1) / hosts / 0.01
         return {
             "impact": impact,
-            "message": "Wrong join strategy - RHS %d; LHS %d" % (rhsRows, lhsRows)
+            "message": "Wrong join strategy - RHS %d; LHS %d" % (rhsRows, lhsRows),
+            "unit": 5
         }
 
 class ExplodingJoinCheck(SQLOperatorReason):
@@ -304,7 +307,8 @@ class ExplodingJoinCheck(SQLOperatorReason):
             impact = probeTime * (rowsReturned - probeRows) / rowsReturned
         return {
             "impact": impact,
-            "message": "Exploding join: %d input rows are exploded to %d output rows" % (probeRows, rowsReturned)
+            "message": "Exploding join: %d input rows are exploded to %d output rows" % (probeRows, rowsReturned),
+            "unit": 5
         }
 
 class NNRpcCheck(SQLOperatorReason):
@@ -327,7 +331,8 @@ class NNRpcCheck(SQLOperatorReason):
         impact = max(0, (totalStorageTime - hdfsRawReadTime) / avgReadThreads)
         return {
             "impact": impact,
-            "message": "This is the time waiting for HDFS NN RPC."
+            "message": "This is the time waiting for HDFS NN RPC.",
+            "unit": 5
         }
 
 class TopDownAnalysis:
@@ -467,7 +472,7 @@ class TopDownAnalysis:
             if isinstance(impact, float) and (impact).is_integer():
               evaluation["impact"] = int(impact)
             if (evaluation["impact"] > 0):
-                reason = models.Reason(message=evaluation['message'], impact=evaluation['impact'])
+                reason = models.Reason(message=evaluation['message'], impact=evaluation['impact'], unit=evaluation['unit'])
                 reasons.append(reason)
         return sorted(reasons, key=lambda x: x.impact, reverse=True)
 
@@ -485,7 +490,7 @@ class TopDownAnalysis:
             if isinstance(impact, float) and (impact).is_integer():
               evaluation["impact"] = int(impact)
             if (evaluation["impact"] > 0):
-                reason = models.Reason(message=evaluation['message'], impact=evaluation['impact'])
+                reason = models.Reason(message=evaluation['message'], impact=evaluation['impact'], unit=evaluation['unit'])
                 reasons.append(reason)
         return sorted(reasons, key=lambda x: x.impact, reverse=True)
 
@@ -555,17 +560,28 @@ class TopDownAnalysis:
           for key, value in stats_mapping.get('Query Timeline').iteritems():
             summary.val.counters.append(models.TCounter(name=value, value=0, unit=5))
 
+          missing_stats = {}
+          for key in ['Tables Missing Stats', 'Tables With Corrupt Table Stats']:
+            if summary.val.info_strings[key]:
+              tables = summary.val.info_strings['Tables Missing Stats'].split(',')
+              for table in tables:
+                missing_stats[table] = 1
+
         def add_host(node, exec_summary_json=exec_summary_json):
           is_plan_node = node.is_plan_node()
           node_id = node.id()
            # Setup Hosts & Broadcast
           if node_id and node.is_regular() and int(node_id) in exec_summary_json:
-
-            node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_json[int(node_id)]["hosts"], unit=0))
+            exec_summary_node = exec_summary_json[int(node_id)]
+            node.val.counters.append(models.TCounter(name='Hosts', value=exec_summary_node["hosts"], unit=0))
             broadcast = 0
             if exec_summary_json[int(node_id)]["broadcast"]:
                 broadcast = 1
             node.val.counters.append(models.TCounter(name='Broadcast', value=broadcast, unit=0))
+            if node.name() == 'HDFS_SCAN_NODE':
+              details = exec_summary_node['detail'].split()
+              node.val.info_strings['Table'] = details[0]
+              node.val.counters.append(models.TCounter(name='MissingStats', value=missing_stats.get(details[0], 0), unit=0))
 
           # Setup LocalTime & ChildTime
           if node_id: