Эх сурвалжийг харах

[metastore] Update the logic to be able to fetch more recent partitions

Romain Rigaux 10 жил өмнө
parent
commit
fff59ed

+ 6 - 4
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -848,20 +848,22 @@ class HiveServerClient:
   def get_partitions(self, database, table_name, partition_spec=None, max_parts=None, reverse_sort=True):
     table = self.get_table(database, table_name)
 
-    if max_parts is None or max_parts <= 0:
-      max_parts = LIST_PARTITIONS_LIMIT.get()
-
     query = 'SHOW PARTITIONS `%s`.`%s`' % (database, table_name)
     if partition_spec:
       query += ' PARTITION(%s)' % partition_spec
 
-    partition_table = self.execute_query_statement(query, max_rows=max_parts)
+    # We fetch N partitions then reverse the order later and get the max_parts. Use partition_spec to refine more the initial list.
+    # Need to fetch more like this until SHOW PARTITIONS offers a LIMIT and ORDER BY
+    partition_table = self.execute_query_statement(query, max_rows=10000)
 
     partitions = [PartitionValueCompatible(partition, table) for partition in partition_table.rows()]
 
     if reverse_sort:
       partitions.reverse()
 
+    if max_parts is None or max_parts <= 0:
+      max_parts = LIST_PARTITIONS_LIMIT.get()
+
     return partitions[:max_parts]
 
 

+ 2 - 2
apps/beeswax/src/beeswax/tests.py

@@ -1668,7 +1668,7 @@ for x in sys.stdin:
       partition_spec = "(baz='baz_one' AND boom='boom_two')"
       table = self.db.get_table(database=self.db_name, table_name=table_name)
       hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
-      assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec), hql)
+      assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec))
     finally:
       finish()
 
@@ -1679,7 +1679,7 @@ for x in sys.stdin:
       partition_spec = "(baz='baz_one' AND boom='boom_two') OR (baz='baz_foo' AND boom='boom_bar')"
       table = self.db.get_table(database=self.db_name, table_name=table_name)
       hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
-      assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec), hql)
+      assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec))
     finally:
       finish()