소스 검색

HUE-2874 [metastore] Sorting the partition list in ascending order doesn't update target links

Jenny Kim 10 년 전
부모
커밋
f36060c

+ 9 - 15
apps/beeswax/src/beeswax/server/dbms.py

@@ -599,29 +599,23 @@ class HiveServer2Dbms(object):
     return self.client.get_partitions(db_name, table.name, partition_spec, max_parts, reverse_sort)
 
 
-  def get_partition(self, db_name, table_name, partition_id):
+  def get_partition(self, db_name, table_name, partition_spec):
     table = self.get_table(db_name, table_name)
-    partitions = self.get_partitions(db_name, table, partition_spec=None, max_parts=None)
+    partitions = self.get_partitions(db_name, table, partition_spec=partition_spec, max_parts=None)
 
-    partition_query = ""
-    for idx, key in enumerate(partitions[partition_id].values):
-      partition_query += (idx > 0 and " AND " or "") + table.partition_keys[idx].name + "='%s'" % key
+    if len(partitions) != 1:
+      raise NoSuchObjectException(_("Query did not return exactly one partition result"))
+
+    partition = partitions[0]
+    partition_query = " AND ".join(partition.partition_spec.split(','))
 
     hql = "SELECT * FROM `%s`.`%s` WHERE %s" % (db_name, table_name, partition_query)
 
     return self.execute_statement(hql)
 
 
-  def describe_partition(self, db_name, table_name, partition_id):
-    table = self.get_table(db_name, table_name)
-    partitions = self.get_partitions(db_name, table, partition_spec=None, max_parts=None)
-
-    parts = ["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[partition_id].values)]
-    partition_spec = ','.join(parts)
-
-    describe_table = self.client.get_table(db_name, table_name, partition_spec=partition_spec)
-
-    return describe_table
+  def describe_partition(self, db_name, table_name, partition_spec):
+    return self.client.get_table(db_name, table_name, partition_spec=partition_spec)
 
 
   def explain(self, query):

+ 5 - 2
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -839,11 +839,14 @@ class PartitionKeyCompatible:
 
 class PartitionValueCompatible:
 
-  def __init__(self, partition, table, properties=None):
+  def __init__(self, partition_row, table, properties=None):
     if properties is None:
       properties = {}
     # Parses: ['datehour=2013022516'] or ['month=2011-07/dt=2011-07-01/hr=12']
-    self.values = [val.split('=')[1] for part in partition for val in part.split('/')]
+    partition = partition_row[0]
+    parts = partition.split('/')
+    self.partition_spec = ','.join(["%s='%s'" % (pv[0], pv[1]) for pv in [part.split('=') for part in parts]])
+    self.values = [pv[1] for pv in [part.split('=') for part in parts]]
     self.sd = type('Sd', (object,), properties,)
 
 

+ 11 - 5
apps/metastore/src/metastore/tests.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 
 import logging
-import json
+import urllib
 
 from nose.tools import assert_true, assert_equal, assert_false
 
@@ -25,9 +25,9 @@ from django.utils.encoding import smart_str
 from django.contrib.auth.models import User, Group
 from django.core.urlresolvers import reverse
 
-import hadoop
 from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod_whitespace
 from desktop.lib.test_utils import add_permission, grant_access
+from hadoop.pseudo_hdfs4 import is_live_cluster
 from useradmin.models import HuePermission, GroupPermission, group_has_permission
 
 from beeswax.conf import BROWSE_PARTITIONED_TABLE_LIMIT
@@ -141,15 +141,21 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
       finish()
 
   def test_read_partitions(self):
-    response = self.client.get("/metastore/table/%s/test_partitions/partitions/0/read" % self.db_name, follow=True)
+    partition_spec = "baz='baz_one',boom='boom_two'"
+    response = self.client.get("/metastore/table/%s/test_partitions/partitions/%s/read" % (self.db_name, partition_spec), follow=True)
     response = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context['query'].id}), follow=True)
     response = wait_for_query_to_finish(self.client, response, max=30.0)
     results = fetch_query_result_data(self.client, response)
     assert_true(len(results['results']) > 0, results)
 
   def test_browse_partition(self):
-    response = self.client.get("/metastore/table/%s/test_partitions/partitions/1/browse" % self.db_name, follow=True)
-    filebrowser_path = reverse("filebrowser.views.view", kwargs={'path': '%s/baz_foo/boom_bar' % self.cluster.fs_prefix})
+    partition_spec = "baz='baz_one',boom='boom_two'"
+    response = self.client.get("/metastore/table/%s/test_partitions/partitions/%s/browse" % (self.db_name, partition_spec), follow=True)
+    if is_live_cluster():
+      path = '/user/hive/warehouse/%s.db/test_partitions/baz=baz_one/boom=boom_two' % self.db_name
+    else:
+      path = '/user/hive/warehouse/test_partitions/baz=baz_one/boom=boom_two'
+    filebrowser_path = urllib.unquote(reverse("filebrowser.views.view", kwargs={'path': path}))
     assert_equal(response.request['PATH_INFO'], filebrowser_path)
 
   def test_drop_multi_tables(self):

+ 2 - 2
apps/metastore/src/metastore/urls.py

@@ -29,6 +29,6 @@ urlpatterns = patterns('metastore.views',
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/load$', 'load_table', name='load_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/read$', 'read_table', name='read_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions$', 'describe_partitions', name='describe_partitions'),
-  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_id>\w+)/read$', 'read_partition', name='read_partition'),
-  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_id>\w+)/browse$', 'browse_partition', name='browse_partition'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_spec>.+?)/read$', 'read_partition', name='read_partition'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions/(?P<partition_spec>.+?)/browse$', 'browse_partition', name='browse_partition'),
 )

+ 13 - 9
apps/metastore/src/metastore/views.py

@@ -17,6 +17,7 @@
 
 import json
 import logging
+import urllib
 
 from django.shortcuts import redirect
 from django.utils.functional import wraps
@@ -278,12 +279,13 @@ def describe_partitions(request, database, table):
   partitions = db.get_partitions(database, table_obj, partition_spec, max_parts=None, reverse_sort=reverse_sort)
 
   massaged_partitions = []
-  for id, partition in enumerate(partitions):
+  for partition in partitions:
     massaged_partitions.append({
-      'id': id,
       'columns': partition.values,
-      'readUrl': reverse('metastore:read_partition', kwargs={'database': database, 'table': table_obj.name, 'partition_id':id}),
-      'browseUrl': reverse('metastore:browse_partition', kwargs={'database': database, 'table': table_obj.name, 'partition_id':id})
+      'readUrl': reverse('metastore:read_partition', kwargs={'database': database, 'table': table_obj.name,
+                                                             'partition_spec': urllib.quote(partition.partition_spec)}),
+      'browseUrl': reverse('metastore:browse_partition', kwargs={'database': database, 'table': table_obj.name,
+                                                                 'partition_spec': urllib.quote(partition.partition_spec)})
     })
 
   if request.method == "POST":
@@ -314,21 +316,23 @@ def describe_partitions(request, database, table):
 
 
 
-def browse_partition(request, database, table, partition_id):
+def browse_partition(request, database, table, partition_spec):
   db = dbms.get(request.user)
   try:
-    partition_table = db.describe_partition(database, table, int(partition_id))
+    decoded_spec = urllib.unquote(partition_spec)
+    partition_table = db.describe_partition(database, table, decoded_spec)
     uri_path = location_to_url(partition_table.path_location)
     return redirect(uri_path)
   except Exception, e:
     raise PopupException(_('Cannot browse partition'), detail=e.message)
 
 
-def read_partition(request, database, table, partition_id):
+def read_partition(request, database, table, partition_spec):
   db = dbms.get(request.user)
   try:
-    partition = db.get_partition(database, table, int(partition_id))
-    url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': partition.id}) + '?on_success_url=&context=table:%s:%s' % (table, database)
+    decoded_spec = urllib.unquote(partition_spec)
+    query = db.get_partition(database, table, decoded_spec)
+    url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query.id}) + '?on_success_url=&context=table:%s:%s' % (table, database)
     return redirect(url)
   except Exception, e:
     raise PopupException(_('Cannot read partition'), detail=e.message)