Эх сурвалжийг харах

HUE-3035 [beeswax] Optimize sample data query for partitioned tables

Uses a configured value, sample_table_max_partitions, to limit the # of partitions that can be read when fetching sample data for partitioned tables
Jenny Kim 10 жил өмнө
parent
commit
787c328

+ 6 - 0
apps/beeswax/src/beeswax/conf.py

@@ -80,6 +80,12 @@ BROWSE_PARTITIONED_TABLE_LIMIT = Config(
   type=int,
   help=_t('Set a LIMIT clause when browsing a partitioned table. A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.'))
 
+SAMPLE_TABLE_MAX_PARTITIONS = Config(
+  key='sample_table_max_partitions',
+  default=10,
+  type=int,
+  help=_t('The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.'))
+
 DOWNLOAD_ROW_LIMIT = Config(
   key='download_row_limit',
   default=1000000,

+ 20 - 8
apps/beeswax/src/beeswax/server/dbms.py

@@ -29,8 +29,8 @@ from desktop.lib.parameterization import substitute_variables
 from filebrowser.views import location_to_url
 
 from beeswax import hive_site
-from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, BROWSE_PARTITIONED_TABLE_LIMIT, SERVER_CONN_TIMEOUT, AUTH_USERNAME, AUTH_PASSWORD, \
-  APPLY_NATURAL_SORT_MAX
+from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, BROWSE_PARTITIONED_TABLE_LIMIT, SERVER_CONN_TIMEOUT, \
+  AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, SAMPLE_TABLE_MAX_PARTITIONS
 from beeswax.common import apply_natural_sort
 from beeswax.design import hql_query
 from beeswax.hive_site import hiveserver2_use_ssl
@@ -265,7 +265,6 @@ class HiveServer2Dbms(object):
     hql = None
 
     if not table.is_view:
-
       limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
 
       if column or nested: # Could do column for any type, then nested with partitions 
@@ -273,11 +272,10 @@ class HiveServer2Dbms(object):
           select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
           hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
       else:
-        partition_query = ""
-        if table.partition_keys:
-          partitions = self.get_partitions(database, table, partition_spec=None, max_parts=1)
-          partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
-        hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)
+        if table.partition_keys:  # Filter on max # of partitions for partitioned tables
+          hql = self._get_sample_partition_query(database, table, limit)
+        else:
+          hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)
 
       if hql:
         query = hql_query(hql)
@@ -290,6 +288,20 @@ class HiveServer2Dbms(object):
     return result
 
 
+  def _get_sample_partition_query(self, database, table, limit):
+    partitions = self.get_partitions(database, table, partition_spec=None, max_parts=SAMPLE_TABLE_MAX_PARTITIONS.get())
+
+    if partitions:
+      # Need to reformat partition specs for where clause syntax
+      partition_specs = [part.partition_spec.replace(',', ' AND ') for part in partitions]
+      partition_filters = ' OR '.join(['(%s)' % partition_spec for partition_spec in partition_specs])
+      partition_clause = 'WHERE %s' % partition_filters
+    else:
+      partition_clause = ''
+
+    return "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_clause, limit)
+
+
   def analyze_table(self, database, table):
     if self.server_name == 'impala':
       hql = 'COMPUTE STATS `%(database)s`.`%(table)s`' % {'database': database, 'table': table}

+ 65 - 0
apps/beeswax/src/beeswax/tests.py

@@ -1651,6 +1651,71 @@ for x in sys.stdin:
     assert_true([0, '0x0'] in resp.context['sample_rows'], resp.context['sample_rows'])
 
 
+  def test_get_sample_partitioned(self):
+    # Test limit of one partition
+    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(1)
+    try:
+      table_name = 'test_partitions'
+      partition_spec = "(baz='baz_one' AND boom='boom_two')"
+      table = self.db.get_table(database=self.db_name, table_name=table_name)
+      hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
+      assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec), hql)
+    finally:
+      finish()
+
+    # Test limit of more than one partition
+    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(2)
+    try:
+      table_name = 'test_partitions'
+      partition_spec = "(baz='baz_one' AND boom='boom_two') OR (baz='baz_foo' AND boom='boom_bar')"
+      table = self.db.get_table(database=self.db_name, table_name=table_name)
+      hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
+      assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec), hql)
+    finally:
+      finish()
+
+    # Test table with non-STRING (INT) partition columns
+    # Also tests for single partition column case
+    hql = """
+      CREATE TABLE test_partitions_int (a INT) PARTITIONED BY (b INT);
+      INSERT OVERWRITE TABLE test_partitions_int PARTITION (b=100)
+        SELECT 101 AS a FROM test_partitions LIMIT 1;
+      INSERT OVERWRITE TABLE test_partitions_int PARTITION (b=200)
+        SELECT 201 AS a FROM test_partitions LIMIT 1;
+      INSERT OVERWRITE TABLE test_partitions_int PARTITION (b=300)
+        SELECT 301 AS a FROM test_partitions LIMIT 1;
+    """
+    resp = _make_query(self.client, hql, database=self.db_name)
+    wait_for_query_to_finish(self.client, resp, max=30.0)
+
+    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(2)
+    try:
+      table_name = 'test_partitions_int'
+      table = self.db.get_table(database=self.db_name, table_name=table_name)
+      result = self.db.get_sample(self.db_name, table)
+      sample = list(result.rows())
+      assert_equal(len(sample), 2, sample)
+    finally:
+      finish()
+
+    # Test table that is partitioned but empty
+    hql = """
+      CREATE TABLE test_partitions_empty (a STRING) PARTITIONED BY (b STRING);
+    """
+    resp = _make_query(self.client, hql, database=self.db_name)
+    wait_for_query_to_finish(self.client, resp, max=30.0)
+
+    finish = conf.SAMPLE_TABLE_MAX_PARTITIONS.set_for_testing(2)
+    try:
+      table_name = 'test_partitions_empty'
+      table = self.db.get_table(database=self.db_name, table_name=table_name)
+      result = self.db.get_sample(self.db_name, table)
+      sample = list(result.rows())
+      assert_equal(len(sample), 0, sample)
+    finally:
+      finish()
+
+
   def test_redacting_queries(self):
     c = make_logged_in_client()
 

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -740,6 +740,9 @@
   # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
   ## browse_partitioned_table_limit=250
 
+  # The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.
+  ## sample_table_max_partitions=10
+
   # A limit to the number of rows that can be downloaded from a query.
   # A value of -1 means there will be no limit.
   # A maximum of 65,000 is applied to XLS downloads.

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -742,6 +742,9 @@
   # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
   ## browse_partitioned_table_limit=250
 
+  # The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.
+  ## sample_table_max_partitions=10
+
   # A limit to the number of rows that can be downloaded from a query.
   # A value of -1 means there will be no limit.
   # A maximum of 65,000 is applied to XLS downloads.