فهرست منبع

HUE-606. Error when browsing a table with too many partitions

Add a new config "browse_partitioned_table_limit", default to 250.
When set to a positive number, this will limit any automatic
"SELECT *" on partitioned tables by adding a LIMIT clause.
If set to 0 or negative, then there is no limit.
bc Wong 14 سال پیش
والد
کامیت
2ddef667c5
3فایلهای تغییر یافته به همراه28 افزوده شده و 2 حذف شده
  1. 6 0
      apps/beeswax/src/beeswax/conf.py
  2. 10 0
      apps/beeswax/src/beeswax/tests.py
  3. 12 2
      apps/beeswax/src/beeswax/views.py

+ 6 - 0
apps/beeswax/src/beeswax/conf.py

@@ -90,3 +90,9 @@ BEESWAX_RUNNING_QUERY_LIFETIME = Config(
   default=604800000L, # 7*24*60*60*1000 (1 week)
   type=long,
   help='Time in seconds for beeswax to persist queries in its cache.')
+
+BROWSE_PARTITIONED_TABLE_LIMIT = Config(
+  key='browse_partitioned_table_limit',
+  default=250,
+  type=int,
+  help='Set a LIMIT clause when browsing a partitioned table. A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.')

+ 10 - 0
apps/beeswax/src/beeswax/tests.py

@@ -112,6 +112,16 @@ class TestBeeswaxWithHadoop(BeeswaxSampleProvider):
     response = self.client.get("/beeswax/table/test/partitions")
     assert_true("is not partitioned." in response.content)
 
+  def test_browse_partitions_with_limit(self):
+    # Limit to 90
+    finish = beeswax.conf.BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
+    try:
+      response = self.client.get("/beeswax/table/test_partitions")
+      assert_true("89" in response.content)
+      assert_false("90" in response.content)
+    finally:
+      finish()
+
   def test_query_with_resource(self):
     script = self.cluster.fs.open("/square.py", "w")
     script.write(

+ 12 - 2
apps/beeswax/src/beeswax/views.py

@@ -43,6 +43,7 @@ from beeswax import common
 from beeswax import data_export
 from beeswax import db_utils
 from beeswax import models
+from beeswax import conf
 
 from jobsub.parameterization import find_variables, substitute_variables
 
@@ -71,7 +72,7 @@ def describe_table(request, table):
   # Don't show samples if it's a view (HUE-526).
   if not is_view:
     # Show the first few rows
-    hql = "SELECT * FROM `%s`" % (table,)
+    hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj))
     query_msg = make_beeswax_query(request, hql)
     try:
       sample_results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
@@ -124,7 +125,8 @@ def drop_table(request, table):
 
 def read_table(request, table):
   """View function for select * from table"""
-  hql = "SELECT * FROM %s" % (table,)
+  table_obj = db_utils.meta_client().get_table("default", table)
+  hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj))
   query_msg = make_beeswax_query(request, hql)
   try:
     return execute_directly(request, query_msg, tablename=table)
@@ -154,6 +156,14 @@ def confirm_query(request, query, on_success_url=None):
     'on_success_url': on_success_url,
   })
 
+def _get_browse_limit_clause(table_obj):
+  """Get the limit clause when browsing a partitioned table"""
+  if table_obj.partitionKeys:
+    limit = conf.BROWSE_PARTITIONED_TABLE_LIMIT.get()
+    if limit > 0:
+      return "LIMIT %d" % (limit,)
+  return ""
+
 
 _SEMICOLON_WHITESPACE = re.compile(";\s*$")
 def _strip_trailing_semicolon(query):