Przeglądaj źródła

HUE-2280 [metastore] Partitions names are not always correct

Jenny Kim 10 lat temu
rodzic
commit
88ba8d5

+ 13 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -576,6 +576,7 @@ class HiveServer2Dbms(object):
 
     return self.client.get_partitions(db_name, table.name, max_parts, reverse_sort)
 
+
   def get_partition(self, db_name, table_name, partition_id):
     table = self.get_table(db_name, table_name)
     partitions = self.get_partitions(db_name, table, max_parts=None)
@@ -589,6 +590,18 @@ class HiveServer2Dbms(object):
     return self.execute_statement(hql)
 
 
+  def describe_partition(self, db_name, table_name, partition_id):
+    table = self.get_table(db_name, table_name)
+    partitions = self.get_partitions(db_name, table, max_parts=None)
+
+    parts = ["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[partition_id].values)]
+    partition_spec = ','.join(parts)
+
+    describe_table = self.client.get_table(db_name, table_name, partition_spec)
+
+    return describe_table
+
+
   def explain(self, query):
     return self.client.explain(query)
 

+ 14 - 7
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -603,14 +603,18 @@ class HiveServerClient:
     return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_NAME',))
 
 
-  def get_table(self, database, table_name):
+  def get_table(self, database, table_name, partition_spec=None):
     req = TGetTablesReq(schemaName=database, tableName=table_name)
     res = self.call(self._client.GetTables, req)
 
     table_results, table_schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT)
     self.close_operation(res.operationHandle)
 
-    query = 'DESCRIBE FORMATTED `%s`.`%s`' % (database, table_name)
+    if partition_spec:
+      query = 'DESCRIBE FORMATTED `%s`.`%s` PARTITION(%s)' % (database, table_name, partition_spec)
+    else:
+      query = 'DESCRIBE FORMATTED `%s`.`%s`' % (database, table_name)
+
     (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000, orientation=TFetchOrientation.FETCH_NEXT)
     self.close_operation(operation_handle)
 
@@ -754,7 +758,8 @@ class HiveServerClient:
     else:
       max_rows = 1000 if max_parts <= 250 else max_parts
 
-    partitionTable = self.execute_query_statement('SHOW PARTITIONS %s.%s' % (database, table_name), max_rows=max_rows)
+    partitionTable = self.execute_query_statement('SHOW PARTITIONS `%s`.`%s`' % (database, table_name), max_rows=max_rows)
+
     partitions = [PartitionValueCompatible(partition, table) for partition in partitionTable.rows()][-max_parts:]
 
     if reverse_sort:
@@ -825,10 +830,12 @@ class PartitionKeyCompatible:
 
 class PartitionValueCompatible:
 
-  def __init__(self, partition, table):
+  def __init__(self, partition, table, properties=None):
+    if properties is None:
+      properties = {}
     # Parses: ['datehour=2013022516'] or ['month=2011-07/dt=2011-07-01/hr=12']
     self.values = [val.split('=')[1] for part in partition for val in part.split('/')]
-    self.sd = type('Sd', (object,), {'location': '%s/%s' % (table.path_location, ','.join(partition)),})
+    self.sd = type('Sd', (object,), properties,)
 
 
 class ExplainCompatible:
@@ -943,8 +950,8 @@ class HiveServerClientCompatible(object):
     return tables
 
 
-  def get_table(self, database, table_name):
-    table = self._client.get_table(database, table_name)
+  def get_table(self, database, table_name, partition_spec=None):
+    table = self._client.get_table(database, table_name, partition_spec)
     return HiveServerTableCompatible(table)
 
 

+ 6 - 8
apps/beeswax/src/beeswax/test_base.py

@@ -356,6 +356,12 @@ class BeeswaxSampleProvider(object):
     """ % (data_file % 1,)
     make_query(cls.client, LOAD_DATA, wait=True, local=False)
 
+    # Insert additional partition data into "test_partitions" table
+    ADD_PARTITION = """
+      ALTER TABLE test_partitions ADD PARTITION(baz='baz_two', boom='boom_two') LOCATION '/tmp/beeswax/baz_two/boom_two'
+    """
+    make_query(cls.client, ADD_PARTITION, wait=True, local=False)
+
     # Create a bunch of other tables
     CREATE_TABLE = """
       CREATE TABLE `%(name)s` (foo INT, bar STRING)
@@ -370,14 +376,6 @@ class BeeswaxSampleProvider(object):
     cls._make_data_file(data_file % 2)
     cls._make_table(table_info['name'], CREATE_TABLE % table_info, data_file % 2)
 
-    # Insert additional partition data into "test_partitions" table
-    INSERT_PARTITION_DATA = """
-      INSERT INTO TABLE test_partitions
-      PARTITION(baz='baz_two', boom='boom_two')
-      SELECT foo, bar FROM test
-    """
-    make_query(cls.client, INSERT_PARTITION_DATA, wait=True, local=False)
-
     # Create a "test_utf8" table.
     table_info = dict(name='test_utf8', comment=cls.get_i18n_table_comment())
     cls._make_i18n_data_file(data_file % 3, 'utf-8')

+ 2 - 11
apps/metastore/src/metastore/templates/describe_partitions.mako

@@ -45,23 +45,14 @@ ${ components.menubar() }
           % for field in table.partition_keys:
               <th>${field.name}</th>
           % endfor
-            <th>${_('Path')}</th>
+              <th>${_('Location')}</th>
           </tr>
           % for partition_id, partition in enumerate(partitions):
             <tr>
             % for idx, key in enumerate(partition.values):
                 <td><a href="${ url('metastore:read_partition', database=database, table=table.name, partition_id=partition_id) }" data-row-selector="true">${key}</a></td>
             % endfor
-            <% location = location_to_url(partition.sd.location) %>
-            % if url:
-                <td data-row-selector-exclude="true">
-                  <a href="${location}">${partition.sd.location}</a>
-                </td>
-            % else:
-                <td>
-                ${partition.sd.location}
-                </td>
-            % endif
+                <td><a href="${ url('metastore:browse_partition', database=database, table=table.name, partition_id=partition_id) }"><i class="fa fa-share-square-o"></i> ${_('View Partition Files')}</a></td>
             </tr>
           % endfor
           </table>

+ 13 - 3
apps/metastore/src/metastore/tests.py

@@ -127,7 +127,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     response = self.client.get("/metastore/table/default/test/partitions", follow=True)
     assert_true("is not partitioned." in response.content)
 
-  def test_browse_partitioned_table_with_limit(self):
+  def test_describe_partitioned_table_with_limit(self):
     # Limit to 90
     finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
     try:
@@ -137,13 +137,23 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     finally:
       finish()
 
-  def test_browse_partitions(self):
-    response = self.client.get("/metastore/table/default/test_partitions/partitions/0", follow=True)
+  def test_read_partitions(self):
+    response = self.client.get("/metastore/table/default/test_partitions/partitions/1/read", follow=True)
     response = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context['query'].id}), follow=True)
     response = wait_for_query_to_finish(self.client, response, max=30.0)
     results = fetch_query_result_data(self.client, response)
     assert_true(len(results['results']) > 0, results)
 
+  def test_browse_partition(self):
+    response = self.client.get("/metastore/table/default/test_partitions/partitions/0/browse", follow=True)
+    filebrowser_path = reverse("filebrowser.views.view", kwargs={'path': '/tmp/beeswax/baz_two/boom_two'})
+    assert_equal(response.request['PATH_INFO'], filebrowser_path)
+
+  def test_describe_partition(self):
+    response = self.client.get("/metastore/table/default/test_partitions/partitions/0")
+    assert_true("Location" in response.content, response.content)
+    assert_true("/tmp/beeswax/baz_two/boom_two" in response.content, response.content)
+
   def test_drop_multi_tables(self):
     hql = """
       CREATE TABLE test_drop_1 (a int);

+ 3 - 2
apps/metastore/src/metastore/urls.py

@@ -26,8 +26,9 @@ urlpatterns = patterns('metastore.views',
   url(r'^tables/(?P<database>\w+)?$', 'show_tables', name='show_tables'),
   url(r'^tables/drop/(?P<database>\w+)$', 'drop_table', name='drop_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)$', 'describe_table', name='describe_table'),
-  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions$', 'describe_partitions', name='describe_partitions'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/load$', 'load_table', name='load_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/read$', 'read_table', name='read_table'),
-  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_id>\w+)$', 'read_partition', name='read_partition'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions$', 'describe_partitions', name='describe_partitions'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_id>\w+)/read$', 'read_partition', name='read_partition'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_id>\w+)/browse$', 'browse_partition', name='browse_partition'),
 )

+ 21 - 10
apps/metastore/src/metastore/views.py

@@ -31,6 +31,7 @@ from beeswax.design import hql_query
 from beeswax.models import SavedQuery, MetaInstall
 from beeswax.server import dbms
 from beeswax.server.dbms import get_query_server_config
+from filebrowser.views import location_to_url
 from metastore.forms import LoadDataForm, DbForm
 from metastore.settings import DJANGO_APPS
 
@@ -217,16 +218,6 @@ def read_table(request, database, table):
     raise PopupException(_('Cannot read table'), detail=e)
 
 
-def read_partition(request, database, table, partition_id):
-  db = dbms.get(request.user)
-  try:
-    partition = db.get_partition(database, table, int(partition_id))
-    url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': partition.id}) + '?on_success_url=&context=table:%s:%s' % (table, database)
-    return redirect(url)
-  except Exception, e:
-    raise PopupException(_('Cannot read table'), detail=e)
-
-
 @check_has_write_access_permission
 def load_table(request, database, table):
   db = dbms.get(request.user)
@@ -289,5 +280,25 @@ def describe_partitions(request, database, table):
   })
 
 
+def browse_partition(request, database, table, partition_id):
+  db = dbms.get(request.user)
+  try:
+    partition_table = db.describe_partition(database, table, int(partition_id))
+    uri_path = location_to_url(partition_table.path_location)
+    return redirect(uri_path)
+  except Exception, e:
+    raise PopupException(_('Cannot browse partition'), detail=e.message)
+
+
+def read_partition(request, database, table, partition_id):
+  db = dbms.get(request.user)
+  try:
+    partition = db.get_partition(database, table, int(partition_id))
+    url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': partition.id}) + '?on_success_url=&context=table:%s:%s' % (table, database)
+    return redirect(url)
+  except Exception, e:
+    raise PopupException(_('Cannot read partition'), detail=e.message)
+
+
 def has_write_access(user):
   return user.is_superuser or user.has_hue_permission(action="write", app=DJANGO_APPS[0])