|
|
@@ -28,8 +28,8 @@ from desktop.lib.parameterization import substitute_variables
|
|
|
from filebrowser.views import location_to_url
|
|
|
|
|
|
from beeswax import hive_site
|
|
|
-from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, BROWSE_PARTITIONED_TABLE_LIMIT, SERVER_CONN_TIMEOUT, \
|
|
|
- AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, SAMPLE_TABLE_MAX_PARTITIONS
|
|
|
+from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
|
|
|
+ AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT
|
|
|
from beeswax.common import apply_natural_sort
|
|
|
from beeswax.design import hql_query
|
|
|
from beeswax.hive_site import hiveserver2_use_ssl
|
|
|
@@ -257,9 +257,8 @@ class HiveServer2Dbms(object):
|
|
|
|
|
|
|
|
|
def select_star_from(self, database, table):
|
|
|
- if table.partition_keys: # Filter on max # of partitions for partitioned tables
|
|
|
- limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
|
|
|
- hql = self._get_sample_partition_query(database, table, limit)
|
|
|
+ if table.partition_keys: # Filter on max number of partitions for partitioned tables
|
|
|
+ hql = self._get_sample_partition_query(database, table, limit=10000) # Currently need a limit
|
|
|
else:
|
|
|
hql = "SELECT * FROM `%s`.`%s`" % (database, table.name)
|
|
|
return self.execute_statement(hql)
|
|
|
@@ -303,35 +302,35 @@ class HiveServer2Dbms(object):
|
|
|
result = None
|
|
|
hql = None
|
|
|
|
|
|
- if not table.is_view:
|
|
|
- limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
|
|
|
+ limit = 100
|
|
|
|
|
|
- if column or nested: # Could do column for any type, then nested with partitions
|
|
|
- if self.server_name == 'impala':
|
|
|
- from impala.dbms import ImpalaDbms
|
|
|
- select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
|
|
|
- hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
|
|
|
+ if column or nested: # Could do column for any type, then nested with partitions
|
|
|
+ if self.server_name == 'impala':
|
|
|
+ from impala.dbms import ImpalaDbms
|
|
|
+ select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
|
|
|
+ hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
|
|
|
+ else:
|
|
|
+ if table.partition_keys: # Filter on max # of partitions for partitioned tables
|
|
|
+ hql = self._get_sample_partition_query(database, table, limit)
|
|
|
else:
|
|
|
- if table.partition_keys: # Filter on max # of partitions for partitioned tables
|
|
|
- hql = self._get_sample_partition_query(database, table, limit)
|
|
|
- else:
|
|
|
- hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)
|
|
|
+ hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)
|
|
|
|
|
|
- if hql:
|
|
|
- query = hql_query(hql)
|
|
|
- handle = self.execute_and_wait(query, timeout_sec=5.0)
|
|
|
+ if hql:
|
|
|
+ query = hql_query(hql)
|
|
|
+ handle = self.execute_and_wait(query, timeout_sec=5.0)
|
|
|
|
|
|
- if handle:
|
|
|
- result = self.fetch(handle, rows=100)
|
|
|
- self.close(handle)
|
|
|
+ if handle:
|
|
|
+ result = self.fetch(handle, rows=100)
|
|
|
+ self.close(handle)
|
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
def _get_sample_partition_query(self, database, table, limit):
|
|
|
- partitions = self.get_partitions(database, table, partition_spec=None, max_parts=SAMPLE_TABLE_MAX_PARTITIONS.get())
|
|
|
+ max_parts = QUERY_PARTITIONS_LIMIT.get()
|
|
|
+ partitions = self.get_partitions(database, table, partition_spec=None, max_parts=max_parts)
|
|
|
|
|
|
- if partitions:
|
|
|
+ if partitions and max_parts:
|
|
|
# Need to reformat partition specs for where clause syntax
|
|
|
partition_specs = [part.partition_spec.replace(',', ' AND ') for part in partitions]
|
|
|
partition_filters = ' OR '.join(['(%s)' % partition_spec for partition_spec in partition_specs])
|
|
|
@@ -712,15 +711,15 @@ class HiveServer2Dbms(object):
|
|
|
|
|
|
|
|
|
def get_partitions(self, db_name, table, partition_spec=None, max_parts=None, reverse_sort=True):
|
|
|
- if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
|
|
|
- max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()
|
|
|
+ if max_parts is None or max_parts > LIST_PARTITIONS_LIMIT.get():
|
|
|
+ max_parts = LIST_PARTITIONS_LIMIT.get()
|
|
|
|
|
|
- return self.client.get_partitions(db_name, table.name, partition_spec, max_parts, reverse_sort)
|
|
|
+ return self.client.get_partitions(db_name, table.name, partition_spec, max_parts=max_parts, reverse_sort=reverse_sort)
|
|
|
|
|
|
|
|
|
def get_partition(self, db_name, table_name, partition_spec):
|
|
|
table = self.get_table(db_name, table_name)
|
|
|
- partitions = self.get_partitions(db_name, table, partition_spec=partition_spec, max_parts=None)
|
|
|
+ partitions = self.get_partitions(db_name, table, partition_spec=partition_spec)
|
|
|
|
|
|
if len(partitions) != 1:
|
|
|
raise NoSuchObjectException(_("Query did not return exactly one partition result"))
|