소스 검색

HUE-2906 [impala] Autocomplete struct nested type
HUE-2907 [impala] Autocomplete helper for map type

Adds an additional beeswax API autocomplete endpoint for nested data types. For nested data types, response will include the type, and the corresponding nested structures (item, key, value, fields).

New API endpoints:
beeswax/api/autocomplete/<database>/<table>/<column>
beeswax/api/autocomplete/<database>/<table>/<column>/<nested_type>/<nested_type>/.../

where <nested_type> token can be:
item : returns the array item type info, if the parent type is an array
key : returns the key type info, if the parent type is a map
value : returns the value type info, if the parent type is a map
<fieldname> : returns the field type info, if the parent type is a struct and fieldname is a field within the struct

All column and nested type responses include:
"type" : simple type-string of the current type (e.g. - "struct")

For arrays, maps, and structs, the additional info is included in the response:
Array:
"item": {type": <simple_type>}
Map:
"key": {type": <simple_type>}
"value": {type": <simple_type>}
Struct:
"fields": [<list of fieldnames>]

Primitive types will only include "type" in the response, where both values are equivalent.

Jenny Kim 10 년 전
부모
커밋
16b4ba5

+ 25 - 88
apps/beeswax/src/beeswax/api.py

@@ -29,6 +29,7 @@ from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 from jobsub.parameterization import substitute_variables
+from metastore import parser
 
 import beeswax.models
 
@@ -42,7 +43,6 @@ from beeswax.views import authorized_get_design, authorized_get_query_history, m
                           safe_get_design, save_design, massage_columns_for_json, _get_query_handle_and_state, \
                           _parse_out_hadoop_jobs
 
-
 LOG = logging.getLogger(__name__)
 
 
@@ -100,20 +100,14 @@ def autocomplete(request, database=None, table=None, column=None, nested=None):
       response['columns'] = [column.name for column in t.cols]
       response['extended_columns'] = massage_columns_for_json(t.cols)
     else:
-      if app_name == 'beeswax':
-        if nested is None:  # autocomplete column
-          t = db.get_table(database, table)
-          current = db.get_column(database, table, column)
-          extended_type, simple_type = _get_column_type_by_name(t.cols, column)
-        else:  # autocomplete nested data type
-          db.use(database)  # Need to explicitly select the DB due to https://issues.apache.org/jira/browse/HIVE-11261
-          current, extended_type, simple_type = _get_nested_describe_and_types(db, database, table, column, nested)
-
-        response['extended_type'] = extended_type
-        response['type'] = simple_type
-
-        inner_type = _get_complex_inner_type(current, extended_type, simple_type)
-        response.update(inner_type)
+      col = db.get_column(database, table, column)
+      if col:
+        parse_tree = parser.parse_column(col.name, col.type, col.comment)
+        if nested:
+          parse_tree = _extract_nested_type(parse_tree, nested)
+        response = parse_tree
+      else:
+        raise Exception('Could not find column `%s`.`%s`.`%s`' % (database, table, column))
   except (QueryServerTimeoutException, TTransportException), e:
     response['code'] = 503
     response['error'] = e.message
@@ -711,77 +705,20 @@ def get_top_terms(request, database, table, column, prefix=None):
 """
 Utils
 """
-def _get_simple_data_type(type_string=None):
-  if type_string:
-    pattern = re.compile('^([a-z]+)(<.+>)?$', re.IGNORECASE)
-    match = re.search(pattern, type_string)
-    return match.group(1)
-  return None
-
-
-def _get_column_type_by_name(columns, column_name):
-  full_type = simple_type = None
-
-  for column in columns:
-    if column.name == column_name:
-      full_type = column.type
-      simple_type = _get_simple_data_type(column.type)
-
-  return full_type, simple_type
-
-
-def _extract_nested_type(type_string, token):
-  full_type = simple_type = None
-
-  if token == '$elem$':
-    full_type = re.search(r"array<(.+)>$", type_string).group(1)
-  elif token == '$key$':
-    full_type = re.search(r"map<(\w+),.+>$", type_string).group(1)
-  elif token == '$value$':
-    full_type= re.search(r"map<\w+,(.+)>$", type_string).group(1)
-
-  if full_type:
-    simple_type = _get_simple_data_type(full_type)
-
-  return full_type, simple_type
-
-
-def _get_parent_describe_and_types(db, database, table, column, nested_tokens):
-  parent_token = nested_tokens[-2] if len(nested_tokens) > 1 else column
-  parent = db.get_column(database, table, column, nested_tokens[:-1])
-  extended_type, simple_type = _get_column_type_by_name(parent.cols, parent_token)
-
-  return parent, extended_type, simple_type
-
-
-def _get_nested_describe_and_types(db, database, table, column, nested):
-  nested_tokens = nested.strip('/').split('/')
-  last_token = nested_tokens[-1]
-  parent, parent_type, parent_simple_type = _get_parent_describe_and_types(db, database, table, column, nested_tokens)
-  current = db.get_column(database, table, column, nested_tokens)
-
-  if last_token in ('$elem$', '$key$', '$value$'):  # ARRAY and MAP types must be parsed from parent_type
-    extended_type, simple_type = _extract_nested_type(parent_type, last_token)
-  else:  # STRUCT or primitive type must be looked up from parent DESCRIBE table
-    extended_type, simple_type = _get_column_type_by_name(parent.cols, last_token)
-
-  return current, extended_type, simple_type
-
-
-def _get_complex_inner_type(current, extended_type, simple_type):
-  inner_type = {}
-
-  # Add nested fields to response
-  if simple_type == 'array':
-    full, short = _extract_nested_type(extended_type, '$elem$')
-    inner_type['elem'] = {'extended_type': full, 'type': short}
-  elif simple_type == 'map':
-    full, short = _extract_nested_type(extended_type, '$key$')
-    inner_type['key'] = {'extended_type': full, 'type': short}
-    full, short = _extract_nested_type(extended_type, '$value$')
-    inner_type['value'] = {'extended_type': full, 'type': short}
-  elif simple_type == 'struct':
-    inner_type['extended_fields'] = massage_columns_for_json(current.cols)
-    inner_type['fields'] = [column.name for column in current.cols]
+def _extract_nested_type(parse_tree, nested_path):
+  nested_tokens = nested_path.strip('/').split('/')
+
+  subtree = parse_tree
+
+  for token in nested_tokens:
+    if token in subtree:
+      subtree = subtree[token]
+    elif 'fields' in subtree:
+      for field in subtree['fields']:
+        if field['name'] == token:
+          subtree = field
+          break
+    else:
+      raise Exception('Invalid nested type path: %s' % nested_path)
 
-  return inner_type
+  return subtree

+ 6 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -156,8 +156,12 @@ class HiveServer2Dbms(object):
     return self.client.get_table(database, table_name)
 
 
-  def get_column(self, database, table_name, column_name, nested_tokens=None):
-    return self.client.get_table(database, table_name, column_name=column_name, nested_tokens=nested_tokens)
+  def get_column(self, database, table_name, column_name):
+    table = self.client.get_table(database, table_name)
+    for col in table.cols:
+      if col.name == column_name:
+        return col
+    return None
 
 
   def execute_query(self, query, design):

+ 5 - 10
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -633,22 +633,17 @@ class HiveServerClient:
     return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_NAME',))
 
 
-  def get_table(self, database, table_name, column_name=None, nested_tokens=None, partition_spec=None):
+  def get_table(self, database, table_name, partition_spec=None):
     req = TGetTablesReq(schemaName=database, tableName=table_name)
     res = self.call(self._client.GetTables, req)
 
     table_results, table_schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT)
     self.close_operation(res.operationHandle)
 
-    if column_name and nested_tokens:  # DESCRIBE on nested types cannot accept database name
-      nested_spec = '.'.join('`%s`' % token for token in nested_tokens)
-      query = 'DESCRIBE FORMATTED `%s`.`%s`.%s' % (table_name, column_name, nested_spec)
+    if partition_spec:
+      query = 'DESCRIBE FORMATTED `%s`.`%s` PARTITION(%s)' % (database, table_name, partition_spec)
     else:
       query = 'DESCRIBE FORMATTED `%s`.`%s`' % (database, table_name)
-      if column_name:
-        query += ' `%s`' % column_name
-      elif partition_spec:
-        query += ' PARTITION(%s)' % partition_spec
 
     (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000, orientation=TFetchOrientation.FETCH_NEXT)
     self.close_operation(operation_handle)
@@ -1008,8 +1003,8 @@ class HiveServerClientCompatible(object):
     return tables
 
 
-  def get_table(self, database, table_name, column_name=None, nested_tokens=None, partition_spec=None):
-    table = self._client.get_table(database, table_name, column_name, nested_tokens, partition_spec)
+  def get_table(self, database, table_name, partition_spec=None):
+    table = self._client.get_table(database, table_name, partition_spec)
     return HiveServerTableCompatible(table)
 
 

+ 5 - 30
apps/beeswax/src/beeswax/tests.py

@@ -59,7 +59,6 @@ import beeswax.models
 import beeswax.views
 
 from beeswax import conf, hive_site
-from beeswax.api import _get_simple_data_type, _extract_nested_type
 from beeswax.conf import HIVE_SERVER_HOST
 from beeswax.views import collapse_whitespace, _save_design
 from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config,\
@@ -1791,20 +1790,17 @@ for x in sys.stdin:
     assert_false('error' in json_resp, 'Failed to autocomplete nested type: %s' % json_resp.get('error'))
 
     assert_equal("array", json_resp['type'])
-    assert_equal("array<struct<bar:int,baz:string>>", json_resp['extended_type'])
-    assert_true("elem" in json_resp)
-    assert_equal("struct", json_resp["elem"]["type"])
-    assert_equal("struct<bar:int,baz:string>", json_resp["elem"]["extended_type"])
+    assert_true("item" in json_resp)
+    assert_equal("struct", json_resp["item"]["type"])
 
-    # Autcomplete nested fields for a given nested type
-    resp = self.client.get(reverse("beeswax:api_autocomplete_nested", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo', 'nested': '$elem$'}))
+    # Autocomplete nested fields for a given nested type
+    resp = self.client.get(reverse("beeswax:api_autocomplete_nested", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo', 'nested': 'item'}))
     json_resp = json.loads(resp.content)
     assert_false('error' in json_resp, 'Failed to autocomplete nested type: %s' % json_resp.get('error'))
 
     assert_equal("struct", json_resp['type'])
-    assert_equal("struct<bar:int,baz:string>", json_resp['extended_type'])
     assert_true("fields" in json_resp)
-    assert_true("extended_fields" in json_resp)
+
 
   def test_databases_quote(self):
     c = self.client
@@ -1819,27 +1815,6 @@ for x in sys.stdin:
       _make_query(c, "DROP DATABASE IF EXISTS `%s`" % db_name, database=self.db_name)
 
 
-def test_beeswax_api_get_simple_data_type():
-  # Return just the outer type name for complex nested types
-  assert_equal('array', _get_simple_data_type('array<struct<first_name:string,last_name:string>>'))
-  # Return same value for primitive types
-  assert_equal('string', _get_simple_data_type('string'))
-  # Return None if None
-  assert_equal(None, _get_simple_data_type())
-
-
-def test_beeswax_api_extract_nested_type():
-  # Return full and simple type of array element if token = $elem$
-  assert_equal(('struct<first_name:string,last_name:string>', 'struct'),
-               _extract_nested_type('array<struct<first_name:string,last_name:string>>', '$elem$'))
-  # Return full and simple type of map key if token = $key$
-  assert_equal(('string', 'string'), _extract_nested_type('map<string,array<string>>', '$key$'))
-  # Return full and simple type of map value if token = $value$
-  assert_equal(('array<string>', 'array'), _extract_nested_type('map<string,array<string>>', '$value$'))
-  # Return None, None if token is not valid
-  assert_equal((None, None), _extract_nested_type('struct<first_name:string,last_name:string>', 'first_name'))
-
-
 def test_import_gzip_reader():
   """Test the gzip reader in create table"""
   # Make gzipped data

+ 114 - 0
apps/metastore/src/metastore/parser.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Helper for parsing a Hive metastore column type
+"""
+import re
+
+
+def parse_column(name, type_string, comment=None):
+  """
+  Returns a dictionary of a Hive column's type metadata and
+   any complex or nested type info
+  """
+  column = {
+    'name': name,
+    'comment': comment or ''
+  }
+  simple_type, inner = _parse_type(type_string)
+  column['type'] = simple_type
+  if inner:
+    column.update(_parse_complex(simple_type, inner))
+  return column
+
+
+def _parse_type(type_string):
+  pattern = re.compile('^([a-z]+)(<(.+)>)?$', re.IGNORECASE)
+  match = re.search(pattern, type_string)
+  return match.group(1), match.group(3)
+
+
+def _parse_complex(simple_type, inner):
+  complex_type = {}
+  if simple_type == "array":
+    complex_type['item'] = _parse_array_item(inner)
+  elif simple_type == "map":
+    complex_type['key'] = _parse_map_key(inner)
+    complex_type['value'] = _parse_map_value(inner)
+  elif simple_type == "struct":
+    complex_type['fields'] = _parse_struct_fields(inner)
+  return complex_type
+
+
+def _parse_array_item(inner):
+  item = {}
+  simple_type, inner = _parse_type(inner)
+  item['type'] = simple_type
+  if inner:
+    item.update(_parse_complex(simple_type, inner))
+  return item
+
+
+def _parse_map_key(inner):
+  key = {}
+  key_type = inner.split(',', 1)[0]
+  key['type'] = key_type
+  return key
+
+
+def _parse_map_value(inner):
+  value = {}
+  value_type = inner.split(',', 1)[1]
+  simple_type, inner = _parse_type(value_type)
+  value['type'] = simple_type
+  if inner:
+    value.update(_parse_complex(simple_type, inner))
+  return value
+
+
+def _parse_struct_fields(inner):
+  fields = []
+  field_tuples = _split_struct_fields(inner)
+  for (name, value) in field_tuples:
+    field = {}
+    field['name'] = name
+    simple_type, inner = _parse_type(value)
+    field['type'] = simple_type
+    if inner:
+      field.update(_parse_complex(simple_type, inner))
+    fields.append(field)
+  return fields
+
+
+def _split_struct_fields(fields_string):
+  fields = []
+  remaining = fields_string
+  while remaining:
+    (fieldname, fieldvalue), remaining = _get_next_struct_field(remaining)
+    fields.append((fieldname, fieldvalue))
+  return fields
+
+
+def _get_next_struct_field(fields_string):
+  fieldname, rest = fields_string.split(':', 1)
+  balanced = 0
+  for pos, char in enumerate(rest):
+    balanced += {'<': 1, '>': -1}.get(char, 0)
+    if balanced == 0 and char in ['>', ',']:
+      return (fieldname, rest[:pos+1].strip(',')), rest[pos+1:]
+  return (fieldname, rest), None

+ 48 - 0
apps/metastore/src/metastore/tests.py

@@ -29,6 +29,7 @@ from django.core.urlresolvers import reverse
 from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod_whitespace
 from desktop.lib.test_utils import add_permission, grant_access
 from hadoop.pseudo_hdfs4 import is_live_cluster
+from metastore import parser
 from useradmin.models import HuePermission, GroupPermission, group_has_permission
 
 from beeswax.conf import BROWSE_PARTITIONED_TABLE_LIMIT
@@ -306,3 +307,50 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
 
     check(client, [200, 302]) # Ok
+
+
+class TestParser(object):
+
+  def test_parse_simple(self):
+    name = 'simple'
+    type = 'string'
+    comment = 'test_parse_simple'
+    column = {'name': name, 'type': type, 'comment': comment}
+    parse_tree = parser.parse_column(name, type, comment)
+    assert_equal(parse_tree, column)
+
+
+  def test_parse_array(self):
+    name = 'array'
+    type = 'array<string>'
+    comment = 'test_parse_array'
+    column = {'name': name, 'type': 'array', 'comment': comment, 'item': {'type': 'string'}}
+    parse_tree = parser.parse_column(name, type, comment)
+    assert_equal(parse_tree, column)
+
+
+  def test_parse_map(self):
+    name = 'map'
+    type = 'map<string,int>'
+    comment = 'test_parse_map'
+    column = {'name': name, 'type': 'map', 'comment': comment, 'key': {'type': 'string'}, 'value': {'type': 'int'}}
+    parse_tree = parser.parse_column(name, type, comment)
+    assert_equal(parse_tree, column)
+
+
+  def test_parse_struct(self):
+    name = 'struct'
+    type = 'struct<name:string,age:int>'
+    comment = 'test_parse_struct'
+    column = {'name': name, 'type': 'struct', 'comment': comment, 'fields': [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'int'}]}
+    parse_tree = parser.parse_column(name, type, comment)
+    assert_equal(parse_tree, column)
+
+
+  def test_parse_nested(self):
+    name = 'nested'
+    type = 'array<struct<name:string,age:int>>'
+    comment = 'test_parse_nested'
+    column = {'name': name, 'type': 'array', 'comment': comment, 'item': {'type': 'struct', 'fields': [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'int'}]}}
+    parse_tree = parser.parse_column(name, type, comment)
+    assert_equal(parse_tree, column)