Bladeren bron

HUE-7639 [dashboard] Support median & percentiles in SQL

jdesjean 8 jaren geleden
bovenliggende
commit
a5bde7f0c5

+ 4 - 0
apps/beeswax/src/beeswax/dashboard_api.py

@@ -25,3 +25,7 @@ LOG = logging.getLogger(__name__)
 
 class HiveDashboardApi(SQLDashboardApi):
   pass
+
+  @classmethod
+  def _supports_median(self):
+    return False

+ 8 - 0
apps/impala/src/impala/dashboard_api.py

@@ -25,3 +25,11 @@ LOG = logging.getLogger(__name__)
 
 class ImpalaDashboardApi(SQLDashboardApi):
   pass
+
+  @classmethod
+  def _supports_median(self):
+    return False
+
+  @classmethod
+  def _supports_percentile(cls):
+    return False

+ 1 - 1
desktop/libs/dashboard/src/dashboard/api.py

@@ -371,7 +371,7 @@ def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_t
     'limit': 10,
     'mincount': 1,
     'isDate': False,
-    'aggregate': {'function': 'unique', 'formula': '', 'plain_formula': '', 'percentiles': [{'value': 50}]}
+    'aggregate': {'function': 'unique', 'formula': '', 'plain_formula': '', 'percentile': 50}
   }
 
   if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'):

+ 1 - 1
desktop/libs/dashboard/src/dashboard/models.py

@@ -42,7 +42,7 @@ NESTED_FACET_FORM = {
     'sort': 'desc',
     'canRange': False,
     'isDate': False,
-    'aggregate': {'function': 'unique', 'formula': '', 'plain_formula': '', 'percentiles': [{'value': 50}]}
+    'aggregate': {'function': 'unique', 'formula': '', 'plain_formula': '', 'percentile': 50}
 }
 
 

+ 1 - 2
desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.ko.js

@@ -1043,8 +1043,7 @@ var Collection = function (vm, collection) {
     facet.properties.facets_form.sort('desc');
 
     facet.properties.facets_form.aggregate.formula('');
-    facet.properties.facets_form.aggregate.percentiles.removeAll();
-    facet.properties.facets_form.aggregate.percentiles.push({'value': 50});
+    facet.properties.facets_form.aggregate.percentile = 50;
 
     if (pivot != null) {
       pivot.aggregate.function.subscribe(function() {

+ 3 - 11
desktop/libs/dashboard/src/dashboard/templates/common_search.mako

@@ -2001,15 +2001,7 @@ ${ dashboard.layout_skeleton(suffix='search') }
       <select data-bind="options: metrics, optionsText: 'label', optionsValue: 'value', value: $data.function, disable: ($parents[1].widgetType() == 'text-facet-widget' && $index() == 0 && !$parent.isFacetForm" class="input-small"></select>
 
       <!-- ko if: $data.function() == 'percentile' -->
-        <!-- ko foreach: percentiles() -->
-          <input type="number" class="input-mini" data-bind="value: value"/>
-          <a href="javascript: void(0)" data-bind="click: function() { $parent.percentiles.remove($data); }">
-            <i class="fa fa-minus" title="${ _('Delete') }"></i>
-          </a>
-        <!-- /ko -->
-        <a href="javascript: void(0)" data-bind="click: function() { percentiles.push(ko.mapping.fromJS({'value': 50})); }">
-          <i class="fa fa-plus" title="${ _('Add') }"></i>
-        </a>
+      <input type="number" class="input-mini" data-bind="value: percentile"/>
       <!-- /ko -->
 
       <select data-bind="options: $root.collection.template.facetFieldsNames, value: $parent.field, optionsCaption: '${ _ko('Field...') }', selectize: $root.collection.template.facetFieldsNames" class="hit-options input-small" style="margin-bottom: 0"></select>
@@ -2801,7 +2793,7 @@ var NUMERIC_HIT_OPTIONS = [
     { value: "min", label: "${ _('Min') }" },
     { value: "max", label: "${ _('Max') }" },
     { value: "median", label: "${ _('Median') }" },
-    { value: "percentile", label: "${ _('Percentiles') }" },
+    { value: "percentile", label: "${ _('Percentile') }" },
     { value: "stddev", label: "${ _('Stddev') }" },
     { value: "variance", label: "${ _('Variance') }" }
 ];
@@ -2812,7 +2804,7 @@ var DATETIME_HIT_OPTIONS = [
     { value: "min", label: "${ _('Min') }" },
     { value: "max", label: "${ _('Max') }" },
     { value: "median", label: "${ _('Median') }" },
-    { value: "percentile", label: "${ _('Percentiles') }" }
+    { value: "percentile", label: "${ _('Percentile') }" }
 ];
 var ALPHA_HIT_COUNTER_OPTIONS = [
     ##{ value: "count", label: "${ _('Group by') }" },

+ 1 - 1
desktop/libs/libsolr/src/libsolr/api.py

@@ -862,7 +862,7 @@ class SolrApi(object):
         f['function'] = 'percentile'
         fields.append('50')
       elif f['function'] == 'percentile':
-        fields.extend(map(lambda a: str(a), [_p['value'] for _p in f['percentiles']]))
+        fields.append(str(f['percentile']))
         f['function'] = 'percentile'
       return '%s(%s)' % (f['function'], ','.join(fields))
 

+ 68 - 6
desktop/libs/notebook/src/notebook/dashboard_api.py

@@ -123,11 +123,34 @@ class SQLDashboardApi(DashboardApi):
             'limit': LIMIT
         }
       elif facet['type'] == 'function': # 1 dim only now
+        aggregate_function = facet['properties']['facets'][0]['aggregate']['function']
+        if (aggregate_function == 'percentile' or aggregate_function == 'median') and not self._supports_percentile() and self._supports_cume_dist():
+          sql_from = '''
+          (SELECT *
+          FROM
+          (
+            SELECT %(field)s, cume_dist() OVER (ORDER BY %(field)s) * 100 AS cume_dist__%(field)s
+            FROM %(database)s.%(table)s
+          ) DEFAULT
+          WHERE cume_dist__%(field)s >= %(value)s) DEFAULT
+          ''' % {
+            'field': facet['properties']['facets'][0]['field'],
+            'value': facet['properties']['facets'][0]['aggregate']['percentile'] if aggregate_function == 'percentile' else 50,
+            'database': database,
+            'table': table
+          }
+        else:
+          sql_from = '%(database)s.%(table)s' % {
+            'database': database,
+            'table': table
+          }
+
         sql = '''SELECT %(fields)s
-        FROM %(database)s.%(table)s
+        FROM %(sql_from)s
         %(filters)s''' % {
             'database': database,
             'table': table,
+            'sql_from': sql_from,
             'fields': self._get_aggregate_function(facet['properties']['facets'][0]),
             'filters': self._convert_filters_to_where(filters),
         }
@@ -398,18 +421,57 @@ class SQLDashboardApi(DashboardApi):
     fields = []
 
     if facet['aggregate']['function'] == 'median':
-      facet['aggregate']['function'] = 'percentile'
-      fields.append('50')
+      if cls._supports_median():
+        facet['aggregate']['function'] = 'MEDIAN'
+        fields.append(facet['field'])
+      elif cls._supports_percentile():
+        facet['aggregate']['function'] = 'PERCENTILE'
+        fields.append('%s, 0.5' % facet['field'])
+      elif cls._supports_cume_dist():
+        facet['aggregate']['function'] = 'MIN'
+        fields.append(facet['field'])
+      else:
+        fields.append(facet['field'])
     elif facet['aggregate']['function'] == 'unique':
-      facet['aggregate']['function'] = 'count'
+      facet['aggregate']['function'] = 'COUNT'
       fields.append('distinct `%(field)s`' % facet)
-    elif facet['aggregate']['function'] == 'percentiles':
-      fields.extend(map(lambda a: str(a), [_p['value'] for _p in facet['aggregate']['percentiles']]))
+    elif facet['aggregate']['function'] == 'percentile':
+      if cls._supports_percentile():
+        fields.append('%s, %s' % (facet['field'], cls._zero_to_one(float(facet['aggregate']['percentile']))))
+      elif cls._supports_cume_dist():
+        facet['aggregate']['function'] = 'MIN'
+        fields.append(facet['field'])
+      else:
+        fields.append(facet['field'])
     else:
       fields.append(facet['field'])
 
     return '%s(%s) ' % (facet['aggregate']['function'], ','.join(fields))
 
+  @classmethod
+  def _zero_to_one(cls, value):
+    if value < 0:
+      return cls._zero_to_one(-1 * value)
+    elif value <= 1:
+      return value
+    else:
+      return value / 100
+
+  @classmethod
+  def _supports_cume_dist(self):
+    return True
+
+  @classmethod
+  def _supports_median(self):
+    return True
+
+  @classmethod
+  def _supports_ntile(self):
+    return True
+
+  @classmethod
+  def _supports_percentile(self):
+    return True
 
   def _get_dimension_field(self, facet):
     # facet salary --> cast(salary / 11000 as INT) * 10 AS salary_range_1