Ver Fonte

[hive] Unify the limit of listed partitions and the limit of partitions in SELECT queries

Romain Rigaux há 10 anos atrás
pai
commit
bde2f0b1d1

+ 15 - 5
apps/beeswax/src/beeswax/conf.py

@@ -74,18 +74,28 @@ USE_GET_LOG_API = Config( # To remove in Hue 4
           'If false, use the FetchResults() thrift call from Hive 1.0 or more instead.')
 )
 
-BROWSE_PARTITIONED_TABLE_LIMIT = Config(
+BROWSE_PARTITIONED_TABLE_LIMIT = Config( # Deprecated, to remove in Hue 4
   key='browse_partitioned_table_limit',
-  default=250,
+  default=1000,
   type=int,
-  help=_t('Set a LIMIT clause when browsing a partitioned table. A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.'))
+  help=_t('Limit the number of partitions to list on the partitions page. A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.'))
 
-SAMPLE_TABLE_MAX_PARTITIONS = Config(
-  key='sample_table_max_partitions',
+QUERY_PARTITIONS_LIMIT = Config(
+  key='query_partitions_limit',
   default=10,
   type=int,
   help=_t('The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.'))
 
+def get_browse_partitioned_table_limit():
+  """Get the old default"""
+  return BROWSE_PARTITIONED_TABLE_LIMIT.get()
+
+LIST_PARTITIONS_LIMIT = Config(
+  key='list_partitions_limit',
+  dynamic_default=get_browse_partitioned_table_limit,
+  type=int,
+  help=_t('Limit the number of partitions that can be listed. A positive value will be set as the LIMIT.'))
+
 DOWNLOAD_ROW_LIMIT = Config(
   key='download_row_limit',
   default=1000000,

+ 27 - 28
apps/beeswax/src/beeswax/server/dbms.py

@@ -28,8 +28,8 @@ from desktop.lib.parameterization import substitute_variables
 from filebrowser.views import location_to_url
 
 from beeswax import hive_site
-from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, BROWSE_PARTITIONED_TABLE_LIMIT, SERVER_CONN_TIMEOUT, \
-  AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, SAMPLE_TABLE_MAX_PARTITIONS
+from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
+  AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT
 from beeswax.common import apply_natural_sort
 from beeswax.design import hql_query
 from beeswax.hive_site import hiveserver2_use_ssl
@@ -257,9 +257,8 @@ class HiveServer2Dbms(object):
 
 
   def select_star_from(self, database, table):
-    if table.partition_keys:  # Filter on max # of partitions for partitioned tables
-      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
-      hql = self._get_sample_partition_query(database, table, limit)
+    if table.partition_keys:  # Filter on max number of partitions for partitioned tables
+      hql = self._get_sample_partition_query(database, table, limit=10000) # Currently need a limit
     else:
       hql = "SELECT * FROM `%s`.`%s`" % (database, table.name)
     return self.execute_statement(hql)
@@ -303,35 +302,35 @@ class HiveServer2Dbms(object):
     result = None
     hql = None
 
-    if not table.is_view:
-      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
+    limit = 100
 
-      if column or nested: # Could do column for any type, then nested with partitions
-        if self.server_name == 'impala':
-          from impala.dbms import ImpalaDbms
-          select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
-          hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
+    if column or nested: # Could do column for any type, then nested with partitions
+      if self.server_name == 'impala':
+        from impala.dbms import ImpalaDbms
+        select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
+        hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
+    else:
+      if table.partition_keys:  # Filter on max # of partitions for partitioned tables
+        hql = self._get_sample_partition_query(database, table, limit)
       else:
-        if table.partition_keys:  # Filter on max # of partitions for partitioned tables
-          hql = self._get_sample_partition_query(database, table, limit)
-        else:
-          hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)
+        hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)
 
-      if hql:
-        query = hql_query(hql)
-        handle = self.execute_and_wait(query, timeout_sec=5.0)
+    if hql:
+      query = hql_query(hql)
+      handle = self.execute_and_wait(query, timeout_sec=5.0)
 
-        if handle:
-          result = self.fetch(handle, rows=100)
-          self.close(handle)
+      if handle:
+        result = self.fetch(handle, rows=100)
+        self.close(handle)
 
     return result
 
 
   def _get_sample_partition_query(self, database, table, limit):
-    partitions = self.get_partitions(database, table, partition_spec=None, max_parts=SAMPLE_TABLE_MAX_PARTITIONS.get())
+    max_parts = QUERY_PARTITIONS_LIMIT.get()
+    partitions = self.get_partitions(database, table, partition_spec=None, max_parts=max_parts)
 
-    if partitions:
+    if partitions and max_parts:
       # Need to reformat partition specs for where clause syntax
       partition_specs = [part.partition_spec.replace(',', ' AND ') for part in partitions]
       partition_filters = ' OR '.join(['(%s)' % partition_spec for partition_spec in partition_specs])
@@ -712,15 +711,15 @@ class HiveServer2Dbms(object):
 
 
   def get_partitions(self, db_name, table, partition_spec=None, max_parts=None, reverse_sort=True):
-    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
-      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()
+    if max_parts is None or max_parts > LIST_PARTITIONS_LIMIT.get():
+      max_parts = LIST_PARTITIONS_LIMIT.get()
 
-    return self.client.get_partitions(db_name, table.name, partition_spec, max_parts, reverse_sort)
+    return self.client.get_partitions(db_name, table.name, partition_spec, max_parts=max_parts, reverse_sort=reverse_sort)
 
 
   def get_partition(self, db_name, table_name, partition_spec):
     table = self.get_table(db_name, table_name)
-    partitions = self.get_partitions(db_name, table, partition_spec=partition_spec, max_parts=None)
+    partitions = self.get_partitions(db_name, table, partition_spec=partition_spec)
 
     if len(partitions) != 1:
       raise NoSuchObjectException(_("Query did not return exactly one partition result"))

+ 3 - 4
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -39,6 +39,7 @@ from TCLIService.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,
 from beeswax import conf as beeswax_conf
 from beeswax import hive_site
 from beeswax.hive_site import hiveserver2_use_ssl
+from beeswax.conf import LIST_PARTITIONS_LIMIT
 from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory
 from beeswax.server.dbms import Table, NoSuchObjectException, DataTable,\
                                 QueryServerException
@@ -848,15 +849,13 @@ class HiveServerClient:
     table = self.get_table(database, table_name)
 
     if max_parts is None or max_parts <= 0:
-      max_rows = 10000
-    else:
-      max_rows = 1000 if max_parts <= 250 else max_parts
+      max_parts = LIST_PARTITIONS_LIMIT.get()
 
     query = 'SHOW PARTITIONS `%s`.`%s`' % (database, table_name)
     if partition_spec:
       query += ' PARTITION(%s)' % partition_spec
 
-    partition_table = self.execute_query_statement(query, max_rows=max_rows)
+    partition_table = self.execute_query_statement(query, max_rows=max_parts)
 
     partitions = [PartitionValueCompatible(partition, table) for partition in partition_table.rows()]
 

+ 4 - 4
apps/beeswax/src/beeswax/tests.py

@@ -1662,7 +1662,7 @@ for x in sys.stdin:
 
   def test_get_sample_partitioned(self):
     # Test limit of one partition
-    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(1)
+    finish = conf.LIST_PARTITIONS_LIMIT.set_for_testing(1)
     try:
       table_name = 'test_partitions'
       partition_spec = "(baz='baz_one' AND boom='boom_two')"
@@ -1673,7 +1673,7 @@ for x in sys.stdin:
       finish()
 
     # Test limit of more than one partition
-    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(2)
+    finish = conf.LIST_PARTITIONS_LIMIT.set_for_testing(2)
     try:
       table_name = 'test_partitions'
       partition_spec = "(baz='baz_one' AND boom='boom_two') OR (baz='baz_foo' AND boom='boom_bar')"
@@ -1696,7 +1696,7 @@ for x in sys.stdin:
     """
     resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
 
-    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(2)
+    finish = conf.LIST_PARTITIONS_LIMIT.set_for_testing(2)
     try:
       table_name = 'test_partitions_int'
       table = self.db.get_table(database=self.db_name, table_name=table_name)
@@ -1712,7 +1712,7 @@ for x in sys.stdin:
     """
     resp = _make_query(self.client, hql, wait=True, local=False, max=60.0, database=self.db_name)
 
-    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(2)
+    finish = conf.LIST_PARTITIONS_LIMIT.set_for_testing(2)
     try:
       table_name = 'test_partitions_empty'
       table = self.db.get_table(database=self.db_name, table_name=table_name)

+ 0 - 1
apps/impala/src/impala/dbms.py

@@ -19,7 +19,6 @@ import logging
 
 from desktop.lib.i18n import smart_str
 
-from beeswax.conf import BROWSE_PARTITIONED_TABLE_LIMIT
 from beeswax.design import hql_query
 from beeswax.models import QUERY_TYPES
 from beeswax.server import dbms

+ 3 - 3
apps/metastore/src/metastore/tests.py

@@ -32,7 +32,7 @@ from hadoop.pseudo_hdfs4 import is_live_cluster
 from metastore import parser
 from useradmin.models import HuePermission, GroupPermission, group_has_permission
 
-from beeswax.conf import BROWSE_PARTITIONED_TABLE_LIMIT
+from beeswax.conf import LIST_PARTITIONS_LIMIT
 from beeswax.views import collapse_whitespace
 from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config, fetch_query_result_data
 from beeswax.models import QueryHistory
@@ -176,7 +176,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
 
   def test_describe_partitioned_table_with_limit(self):
     # We have 2 partitions in the test table
-    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("1")
+    finish = LIST_PARTITIONS_LIMIT.set_for_testing("1")
     try:
       response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
       partition_values_json = json.loads(response.context['partition_values_json'])
@@ -184,7 +184,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     finally:
       finish()
 
-    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("3")
+    finish = LIST_PARTITIONS_LIMIT.set_for_testing("3")
     try:
       response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
       partition_values_json = json.loads(response.context['partition_values_json'])

+ 2 - 2
apps/metastore/src/metastore/views.py

@@ -244,7 +244,7 @@ def describe_table(request, database, table):
 
     partitions = None
     if app_name != 'impala' and table.partition_keys:
-      partitions = db.get_partitions(database, table, partition_spec=None, max_parts=None)
+      partitions = db.get_partitions(database, table)
 
     return render(renderable, request, {
       'breadcrumbs': [{
@@ -416,7 +416,7 @@ def describe_partitions(request, database, table):
   else:
     partition_spec = ''
 
-  partitions = db.get_partitions(database, table_obj, partition_spec, max_parts=None, reverse_sort=reverse_sort)
+  partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
 
   massaged_partitions = []
   for partition in partitions:

+ 3 - 4
desktop/conf.dist/hue.ini

@@ -754,12 +754,11 @@
   # If false, use the FetchResults() thrift call from Hive 1.0 or more instead.
   ## use_get_log_api=false
 
-  # Set a LIMIT clause when browsing a partitioned table.
-  # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
-  ## browse_partitioned_table_limit=250
+  # Limit the number of partitions that can be listed.
+  ## list_partitions_limit=10000
 
   # The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.
-  ## sample_table_max_partitions=10
+  ## query_partitions_limit=10
 
   # A limit to the number of rows that can be downloaded from a query.
   # A value of -1 means there will be no limit.

+ 3 - 4
desktop/conf/pseudo-distributed.ini.tmpl

@@ -756,12 +756,11 @@
   # If false, use the FetchResults() thrift call from Hive 1.0 or more instead.
   ## use_get_log_api=false
 
-  # Set a LIMIT clause when browsing a partitioned table.
-  # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
-  ## browse_partitioned_table_limit=250
+  # Limit the number of partitions that can be listed.
+  ## list_partitions_limit=10000
 
   # The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.
-  ## sample_table_max_partitions=10
+  ## query_partitions_limit=10
 
   # A limit to the number of rows that can be downloaded from a query.
   # A value of -1 means there will be no limit.