Browse Source

HUE-6131 [hive] Select partition values based on the actual datatypes of the partition column

Jenny Kim 8 years ago
parent
commit
1e82215132

+ 11 - 1
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -1117,19 +1117,29 @@ class PartitionKeyCompatible:
 class PartitionValueCompatible:
 
   def __init__(self, partition_row, table, properties=None):
+    self.partition_keys = table.partition_keys
     if properties is None:
       properties = {}
     # Parses: ['datehour=2013022516'] or ['month=2011-07/dt=2011-07-01/hr=12']
     partition = partition_row[0]
     parts = partition.split('/')
-    self.partition_spec = ','.join(["`%s`='%s'" % (pv[0], pv[1]) for pv in [part.split('=') for part in parts]])
+    self.partition_spec = ','.join([self._get_partition_spec(pv[0], pv[1]) for pv in [part.split('=') for part in parts]])
     self.values = [pv[1] for pv in [part.split('=') for part in parts]]
     self.sd = type('Sd', (object,), properties,)
 
+
   def __repr__(self):
     return 'PartitionValueCompatible(spec:%s, values:%s, sd:%s)' % (self.partition_spec, self.values, self.sd)
 
 
+  def _get_partition_spec(self, name, value):
+    partition_spec = "`%s`='%s'" % (name, value)
+    partition_key = next((key for key in self.partition_keys if key.name == name), None)
+    if partition_key and partition_key.type.upper() not in ('STRING', 'CHAR', 'VARCHAR', 'TIMESTAMP', 'DATE'):
+      partition_spec = "`%s`=%s" % (name, value)
+    return partition_spec
+
+
 class ExplainCompatible:
 
   def __init__(self, data_table):

+ 3 - 3
apps/beeswax/src/beeswax/test_base.py

@@ -437,7 +437,7 @@ class BeeswaxSampleProvider(object):
       # Create a "test_partitions" table.
       CREATE_PARTITIONED_TABLE = """
         CREATE TABLE `%(db)s`.`test_partitions` (foo INT, bar STRING)
-        PARTITIONED BY (baz STRING, boom STRING)
+        PARTITIONED BY (baz STRING, boom INT)
         ROW FORMAT DELIMITED
           FIELDS TERMINATED BY '\t'
           LINES TERMINATED BY '\n'
@@ -448,13 +448,13 @@ class BeeswaxSampleProvider(object):
       LOAD_DATA = """
         LOAD DATA INPATH '%(data_file)s'
         OVERWRITE INTO TABLE `%(db)s`.`test_partitions`
-        PARTITION (baz='baz_one', boom='boom_two')
+        PARTITION (baz='baz_one', boom=12345)
       """ % {'db': cls.db_name, 'data_file': data_file % 1}
       make_query(cls.client, LOAD_DATA, wait=True, local=False)
 
       # Insert additional partition data into "test_partitions" table
       ADD_PARTITION = """
-        ALTER TABLE `%(db)s`.`test_partitions` ADD PARTITION(baz='baz_foo', boom='boom_bar') LOCATION '%(fs_prefix)s/baz_foo/boom_bar'
+        ALTER TABLE `%(db)s`.`test_partitions` ADD PARTITION(baz='baz_foo', boom=67890) LOCATION '%(fs_prefix)s/baz_foo/boom_bar'
       """ % {'db': cls.db_name, 'fs_prefix': cls.cluster.fs_prefix}
       make_query(cls.client, ADD_PARTITION, wait=True, local=False)
 

+ 2 - 2
apps/beeswax/src/beeswax/tests.py

@@ -1761,7 +1761,7 @@ for x in sys.stdin:
     finish = conf.QUERY_PARTITIONS_LIMIT.set_for_testing(1)
     try:
       table_name = 'test_partitions'
-      partition_spec = "(`baz`='baz_one' AND `boom`='boom_two')"
+      partition_spec = "(`baz`='baz_one' AND `boom`=12345)"
       table = self.db.get_table(database=self.db_name, table_name=table_name)
       hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
       assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec))
@@ -1772,7 +1772,7 @@ for x in sys.stdin:
     finish = conf.QUERY_PARTITIONS_LIMIT.set_for_testing(2)
     try:
       table_name = 'test_partitions'
-      partition_spec = "(`baz`='baz_one' AND `boom`='boom_two') OR (`baz`='baz_foo' AND `boom`='boom_bar')"
+      partition_spec = "(`baz`='baz_one' AND `boom`=12345) OR (`baz`='baz_foo' AND `boom`=67890)"
       table = self.db.get_table(database=self.db_name, table_name=table_name)
       hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
       assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec))

+ 9 - 9
apps/metastore/src/metastore/tests.py

@@ -169,9 +169,9 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     data = json.loads(response.content)
     partition_columns = [col for cols in data['partition_values_json'] for col in cols['columns']]
     assert_true("baz_one" in partition_columns)
-    assert_true("boom_two" in partition_columns)
+    assert_true('12345' in partition_columns, partition_columns)
     assert_true("baz_foo" in partition_columns)
-    assert_true("boom_bar" in partition_columns)
+    assert_true('67890' in partition_columns)
 
     # Not partitioned
     response = self.client.get("/metastore/table/%s/test/partitions" % self.db_name, follow=True)
@@ -199,7 +199,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     if not is_live_cluster():
       raise SkipTest
 
-    partition_spec = "baz='baz_one',boom='boom_two'"
+    partition_spec = "baz='baz_one',boom=12345"
     response = self.client.get("/metastore/table/%s/test_partitions/partitions/%s/read" % (self.db_name, partition_spec), follow=True)
     response = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context['query'].id}), follow=True)
     response = wait_for_query_to_finish(self.client, response, max=30.0)
@@ -207,18 +207,18 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     assert_true(len(results['results']) > 0, results)
 
   def test_browse_partition(self):
-    partition_spec = "baz='baz_one',boom='boom_two'"
+    partition_spec = "baz='baz_one',boom=12345"
     response = self.client.get("/metastore/table/%s/test_partitions/partitions/%s/browse" % (self.db_name, partition_spec), follow=True)
     if is_live_cluster():
-      path = '/user/hive/warehouse/%s.db/test_partitions/baz=baz_one/boom=boom_two' % self.db_name
+      path = '/user/hive/warehouse/%s.db/test_partitions/baz=baz_one/boom=12345' % self.db_name
     else:
-      path = '/user/hive/warehouse/test_partitions/baz=baz_one/boom=boom_two'
+      path = '/user/hive/warehouse/test_partitions/baz=baz_one/boom=12345'
     filebrowser_path = urllib.unquote(reverse("filebrowser.views.view", kwargs={'path': path}))
     assert_equal(response.request['PATH_INFO'], filebrowser_path)
 
   def test_drop_partition(self):
     # Create partition first
-    partition_spec = "baz='baz_drop',boom='boom_drop'"
+    partition_spec = "baz='baz_drop',boom=54321"
     hql = 'ALTER TABLE `%s`.`test_partitions` ADD IF NOT EXISTS PARTITION (%s);' % (self.db_name, partition_spec)
     resp = _make_query(self.client, hql, database=self.db_name)
     wait_for_query_to_finish(self.client, resp, max=30.0)
@@ -330,9 +330,9 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     assert_equal_mod_whitespace("LOAD DATA INPATH '%(data_path)s' INTO TABLE `%(db)s`.`test`" % {'data_path': data_path, 'db': self.db_name}, query.query)
 
     # Try it with partitions
-    resp = self.client.post("/metastore/table/%s/test_partitions/load" % self.db_name, {'path': data_path, 'partition_0': "alpha", 'partition_1': "beta"}, follow=True)
+    resp = self.client.post("/metastore/table/%s/test_partitions/load" % self.db_name, {'path': data_path, 'partition_0': "alpha", 'partition_1': 12345}, follow=True)
     query = QueryHistory.objects.latest('id')
-    assert_equal_mod_whitespace(query.query, "LOAD DATA INPATH '%(data_path)s' INTO TABLE `%(db)s`.`test_partitions` PARTITION (baz='alpha', boom='beta')" % {'data_path': data_path, 'db': self.db_name})
+    assert_equal_mod_whitespace(query.query, "LOAD DATA INPATH '%(data_path)s' INTO TABLE `%(db)s`.`test_partitions` PARTITION (baz='alpha', boom='12345')" % {'data_path': data_path, 'db': self.db_name})
 
 
   def test_has_write_access_frontend(self):