Browse Source

HUE-2958 [impala] Autocomplete sample of values for scalar columns and nested types

Supported only in Impala mode, and provides a sample of values for a given scalar column or type, as a "sample" field.

Examples:

/impala/api/autocomplete/default/customers/id

{
    "comment": "",
    "sample": [
        10271,
        10326,
        12532,
        13106
    ],
    "type": "int",
    "name": "id"
}

/impala/api/autocomplete/default/customers/orders/item/items/item/product_id

{
    "sample": [
        9661724,
        9705047,
        9741735,
        9767975,
        9976715
    ],
    "type": "int",
    "name": "product_id"
}
Jenny Kim 10 years ago
parent
commit
0fb5dbf391

+ 7 - 0
apps/beeswax/src/beeswax/api.py

@@ -115,6 +115,13 @@ def _autocomplete(db, database=None, table=None, column=None, nested=None):
         if nested:
           parse_tree = _extract_nested_type(parse_tree, nested)
         response = parse_tree
+        # If column or nested type is scalar/primitive, add sample of values
+        if parser.is_scalar_type(parse_tree['type']):
+          table_obj = db.get_table(database, table)
+          sample = db.get_sample(database, table_obj, column, nested)
+          if sample:
+            sample = set([row[0] for row in db.get_sample(database, table_obj, column, nested).rows()])
+            response['sample'] = sorted(list(sample))
       else:
         raise Exception('Could not find column `%s`.`%s`.`%s`' % (database, table, column))
   except (QueryServerTimeoutException, TTransportException), e:

+ 84 - 13
apps/beeswax/src/beeswax/server/dbms.py

@@ -150,6 +150,39 @@ class HiveServer2Dbms(object):
     return cleaned
 
 
+  @classmethod
+  def get_impala_nested_select(cls, database, table, column, nested=None):
+    """
+    Given a column or nested type, return the corresponding SELECT and FROM clauses in Impala's nested-type syntax
+    """
+    select_tokens = [column]
+    from_tokens = [database, table]
+
+    if nested:
+      nested_tokens = nested.strip('/').split('/')
+      while nested_tokens:
+        token = nested_tokens.pop(0)
+        if token not in ['key', 'value', 'item']:
+          select_tokens.append(token)
+        else:
+          # if we encounter a reserved keyword, move current select_tokens to from_tokens and reset the select_tokens
+          from_tokens.extend(select_tokens)
+          select_tokens = []
+          # if reserved keyword is the last token, make it the only select_token, otherwise we ignore and continue
+          if not nested_tokens:
+            select_tokens = [token]
+
+    select_clause = '.'.join(select_tokens)
+    from_clause = '.'.join('`%s`' % token for token in from_tokens)
+    return select_clause, from_clause
+
+
+  @classmethod
+  def get_histogram_query(cls, database, table, column, nested=None):
+    select_clause, from_clause = cls.get_impala_nested_select(database, table, column, nested)
+    return 'SELECT histogram(%s) FROM %s' % (select_clause, from_clause)
+
+
   def get_databases(self, database_names='*'):
     identifier = self.to_matching_wildcard(database_names)
 
@@ -215,6 +248,33 @@ class HiveServer2Dbms(object):
     return None
 
 
+  def get_histogram(self, database, table, column, nested=None):
+    """
+    Returns the results of an Impala SELECT histogram() FROM query for a given column or nested type.
+
+    Assumes that the column/nested type is scalar.
+    """
+    results = []
+
+    if self.server_name == 'impala':  # Currently histogram() is only supported by Impala
+      hql = self.get_histogram_query(database, table, column, nested)
+      query = hql_query(hql)
+      handle = self.execute_and_wait(query, timeout_sec=5.0)
+
+      if handle:
+        result = self.fetch(handle)
+        try:
+          histogram = list(result.rows())[0][0]  # actual histogram results is in first-and-only result row
+          unique_values = set(histogram.split(', '))
+          results = list(unique_values)
+        except IndexError, e:
+          LOG.warn('Failed to get histogram results, result set has unexpected format: %s' % smart_str(e))
+        finally:
+          self.close(handle)
+
+    return results
+
+
   def execute_query(self, query, design):
     return self.execute_and_watch(query, design=design)
 
@@ -258,22 +318,33 @@ class HiveServer2Dbms(object):
     return resp
 
 
-  def get_sample(self, database, table):
-    """No samples if it's a view (HUE-526)"""
+  def get_sample(self, database, table, column=None, nested=None):
+    result = None
+
+    # No samples if it's a view (HUE-526)
     if not table.is_view:
+
       limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
-      partition_query = ""
-      if table.partition_keys:
-        partitions = self.get_partitions(database, table, partition_spec=None, max_parts=1)
-        partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
-      hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)
-      query = hql_query(hql)
-      handle = self.execute_and_wait(query, timeout_sec=5.0)
 
-      if handle:
-        result = self.fetch(handle, rows=100)
-        self.close(handle)
-        return result
+      if (column or nested) and self.server_name == 'impala':  # SELECT column or nested type
+          select_clause, from_clause = self.get_impala_nested_select(database, table.name, column, nested)
+          hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
+      elif not column and not nested:  # SELECT * FROM table
+        partition_query = ""
+        if table.partition_keys:
+          partitions = self.get_partitions(database, table, partition_spec=None, max_parts=1)
+          partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
+        hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)
+
+      if hql:
+        query = hql_query(hql)
+        handle = self.execute_and_wait(query, timeout_sec=5.0)
+
+        if handle:
+          result = self.fetch(handle, rows=100)
+          self.close(handle)
+
+    return result
 
 
   def analyze_table(self, database, table):

+ 16 - 0
apps/beeswax/src/beeswax/tests.py

@@ -3030,3 +3030,19 @@ def test_apply_natural_sort():
                                                             {'name': 'test_2', 'comment': 'Test'},
                                                             {'name': 'test_100', 'comment': 'Test'},
                                                             {'name': 'test_200', 'comment': 'Test'}])
+
+
+def test_get_impala_nested_select():
+  select_fn = dbms.HiveServer2Dbms.get_impala_nested_select
+
+  assert_equal(select_fn('default', 'customers', 'id', None), ('id', '`default`.`customers`'))
+  assert_equal(select_fn('default', 'customers', 'email_preferences', 'categories/promos/'),
+               ('email_preferences.categories.promos', '`default`.`customers`'))
+  assert_equal(select_fn('default', 'customers', 'addresses', 'key'),
+               ('key', '`default`.`customers`.`addresses`'))
+  assert_equal(select_fn('default', 'customers', 'addresses', 'value/street_1/'),
+               ('street_1', '`default`.`customers`.`addresses`'))
+  assert_equal(select_fn('default', 'customers', 'orders', 'item/order_date'),
+               ('order_date', '`default`.`customers`.`orders`'))
+  assert_equal(select_fn('default', 'customers', 'orders', 'item/items/item/product_id'),
+               ('product_id', '`default`.`customers`.`orders`.`items`'))

+ 4 - 0
apps/metastore/src/metastore/parser.py

@@ -37,6 +37,10 @@ def parse_column(name, type_string, comment=None):
   return column
 
 
+def is_scalar_type(type_string):
+  return not (type_string.startswith('array') or type_string.startswith('map') or type_string.startswith('struct'))
+
+
 def _parse_type(type_string):
   pattern = re.compile('^([a-z]+)(<(.+)>)?$', re.IGNORECASE)
   match = re.search(pattern, type_string)