Преглед на файлове

HUE-3228 [dashboard] Support Kudu timestamps and drop bigint as date option

Romain Rigaux преди 8 години
родител
ревизия
6e727b5

+ 2 - 2
README.md

@@ -9,8 +9,8 @@ Hue is an open source Query Tool for browsing, querying and visualizing data wit
 It features:
 
    * [Editors](http://gethue.com/sql-editor/) for Hive, Impala, Pig, MapReduce, Spark and any SQL like MySQL, Oracle, SparkSQL, Solr SQL, Phoenix and more.
-   * [Dashboards](http://gethue.com/search-dashboards/) to dynamically interact and visualize data with Solr or SQL
-   * [Scheduler](http://gethue.com/scheduling/) of jobs and workflows
+   * [Dashboards](http://gethue.com/search-dashboards/) to dynamically interact and visualize data with Solr or SQL.
+   * [Scheduler](http://gethue.com/scheduling/) of jobs and workflows.
    * [Browsers](http://gethue.com/browsers/) for Jobs, HDFS, S3 files, SQL Tables, Indexes, Git files, Sentry permissions, Sqoop and more.
 
 

+ 7 - 17
apps/impala/src/impala/dashboard_api.py

@@ -206,7 +206,6 @@ class SQLApi():
     if result:
       stats = list(result['data'])
       min_value, max_value = stats[0]
-      maybe_is_big_int_date = isinstance(min_value, (int, long))
 
       if not isinstance(min_value, numbers.Number):
         min_value = min_value.replace(' ', 'T') + 'Z'
@@ -218,9 +217,6 @@ class SQLApi():
             fields[0]: {
               'min': min_value,
               'max': max_value,
-              'min_date_if_bigint': datetime.fromtimestamp(min_value).strftime('%Y-%m-%dT%H:%M:%SZ') if maybe_is_big_int_date else min_value,
-              'max_date_if_bigint': datetime.fromtimestamp(max_value).strftime('%Y-%m-%dT%H:%M:%SZ') if maybe_is_big_int_date else max_value,
-              'maybe_is_big_int_date': maybe_is_big_int_date
             }
           }
         }
@@ -357,9 +353,6 @@ class SQLApi():
           quote = "'"
         else:
           quote = ''
-          if  any([c['properties'].get('isBigIntDate') for c in collection['facets'] if c['field'] == fq['field']]):
-            fq['properties'][0]['from'] = "unix_timestamp('%(from)s')" % fq['properties'][0]
-            fq['properties'][0]['to'] = "unix_timestamp('%(to)s')" % fq['properties'][0]
         clauses.append("`%(field)s` >= %(quote)s%(from)s%(quote)s AND `%(field)s` < %(quote)s%(to)s%(quote)s" % {
           'field': fq['field'],
           'to': fq['properties'][0]['to'],
@@ -410,10 +403,7 @@ class SQLApi():
       field_name = '%(field)s_range' % facet
       order_by = '`%(field)s_range` ASC' % facet
       if facet['properties']['isDate']:
-        if facet['properties']['isBigIntDate']:
-          field = 'cast(`%(field)s` AS timestamp)' % facet
-        else:
-          field = '`%(field)s`' % facet
+        field = '`%(field)s`' % facet
 
         slot = self._gap_to_units(facet['properties']['gap'])
 
@@ -456,6 +446,7 @@ class SQLApi():
 
   def _gap_to_units(self, gap):
     skip, coeff, unit = re.split('(\d+)', gap.strip('+')) # e.g. +1HOURS
+
     duration = {
       'coeff': int(coeff),
       'unit': unit.rstrip('S'),
@@ -472,11 +463,11 @@ class SQLApi():
       duration['sql_trunc'] = 'HH'
       duration['sql_interval'] = '1 HOUR'
       duration['timedelta'] = timedelta(seconds=60 * 60)
-    elif duration['unit'] == 'DAY'  and duration['coeff'] == 1:
+    elif duration['unit'] == 'DAY' and duration['coeff'] == 1:
       duration['sql_trunc'] = 'DD'
       duration['sql_interval'] = '1 DAY'
       duration['timedelta'] = timedelta(days=1)
-    elif duration['unit'] == 'WEEK':
+    elif duration['unit'] == 'WEEK' or (duration['unit'] == 'DAY' and duration['coeff'] == 7):
       duration['sql_trunc'] = 'WW'
       duration['sql_interval'] = '1 WEEK'
       duration['timedelta'] = timedelta(days=7)
@@ -493,6 +484,9 @@ class SQLApi():
       duration['sql_interval'] = '1 YEAR'
       duration['timedelta'] = timedelta(days=365)
 
+    if not duration['sql_trunc']:
+      LOG.warn('Duration %s not converted to SQL buckets.' % duration)
+
     return duration
 
   def _get_field(self, collection, name):
@@ -531,10 +525,6 @@ class SQLApi():
         props['from'] = "now() - interval %(coeff)s %(unit)s" % duration
         props['to'] = 'now()' # TODO +/- Proper Tz of user
 
-        if any([c['properties'].get('isBigIntDate') for c in collection['facets'] if c['field'] == time_field]):
-          props['from'] = 'unix_timestamp(%(from)s)' % props
-          props['to'] = 'unix_timestamp(%(to)s)' % props
-
       elif collection['timeFilter']['type'] == 'fixed':
         props['from'] = collection['timeFilter'].get('from', 'now() - interval 7 DAY')
         props['to'] = collection['timeFilter'].get('to', 'now()')

+ 0 - 1
desktop/libs/dashboard/src/dashboard/api.py

@@ -366,7 +366,6 @@ def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_t
     'limit': 10,
     'mincount': 0,
     'isDate': False,
-    'isBigIntDate': False,
     'aggregate': {'function': 'unique', 'ops': [], 'percentiles': [{'value': 50}]}
   }
 

+ 0 - 6
desktop/libs/dashboard/src/dashboard/facet_builder.py

@@ -55,11 +55,6 @@ def _compute_range_facet(widget_type, stat_facet, properties, start=None, end=No
       SLOTS = 100
       
     is_date = widget_type == 'timeline-widget'
-    is_big_int_date = is_date and stat_facet.get('maybe_is_big_int_date')
-
-    if is_big_int_date:
-      stat_facet['min'] = stat_facet['min_date_if_bigint']
-      stat_facet['max'] = stat_facet['max_date_if_bigint']
 
     if isinstance(stat_facet['min'], numbers.Number):
       stats_min = int(stat_facet['min']) # Cast floats to int currently
@@ -159,7 +154,6 @@ def _compute_range_facet(widget_type, stat_facet, properties, start=None, end=No
       'gap': gap,
       'canRange': True,
       'isDate': is_date,
-      'isBigIntDate': is_big_int_date,
     })
 
     if widget_type == 'histogram-widget':

+ 1 - 1
desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.ko.js

@@ -1482,7 +1482,7 @@ var QueryResult = function (vm, initial) { // Similar to to Notebook Snippet
 };
 
 
-var DATE_TYPES = ['date', 'tdate', 'timestamp', 'bigint'];
+var DATE_TYPES = ['date', 'tdate', 'timestamp'];
 var NUMBER_TYPES = ['int', 'tint', 'long', 'tlong', 'float', 'tfloat', 'double', 'tdouble', 'currency'];
 var FLOAT_TYPES = ['float', 'tfloat', 'double', 'tdouble'];
 var GEO_TYPES = ['SpatialRecursivePrefixTreeFieldType'];