Bladeren bron

No ticket. Bump up beeswax_server_thrift_timeout because users are observing that
expensive queries are timing out.
Also making beeswax_server_thrift_timeout and metastore_thrift_timeout configurable.

vinithra 15 jaren geleden
bovenliggende
commit
3e4e45526e
3 gewijzigde bestanden met toevoegingen van 23 en 8 verwijderingen
  1. 8 0
      apps/beeswax/conf/hue-beeswax.ini
  2. 12 0
      apps/beeswax/src/beeswax/conf.py
  3. 3 8
      apps/beeswax/src/beeswax/db_utils.py

+ 8 - 0
apps/beeswax/conf/hue-beeswax.ini

@@ -14,3 +14,11 @@
 #
 # Hive configuration directory, where hive-site.xml is located
 ## hive_conf_dir=/etc/hive/conf
+
+#
+# Timeout in seconds for thrift calls to beeswax service
+## beeswax_server_conn_timeout=120
+
+#
+# Timeout in seconds for thrift calls to the hive metastore
+## metastore_conn_timeout=10

+ 12 - 0
apps/beeswax/src/beeswax/conf.py

@@ -65,3 +65,15 @@ LOCAL_EXAMPLES_DATA_DIR = Config(
   key='local_examples_data_dir',
   default=os.path.join(os.path.dirname(__file__), "..", "..", "data"),
   help='The local filesystem path containing the beeswax examples')
+
+BEESWAX_SERVER_CONN_TIMEOUT = Config(
+  key='beeswax_server_conn_timeout',
+  default=120,
+  type=int,
+  help='Timeout in seconds for thrift calls to beeswax service')
+
+METASTORE_CONN_TIMEOUT= Config(
+  key='metastore_conn_timeout',
+  default=10,
+  type=int,
+  help='Timeouts in seconds for thrift calls to the hive metastore. This timeout should take into account that the metastore could talk to an external DB')

+ 3 - 8
apps/beeswax/src/beeswax/db_utils.py

@@ -35,12 +35,6 @@ from beeswaxd.ttypes import BeeswaxException, QueryHandle, QueryNotFoundExceptio
 
 LOG = logging.getLogger(__name__)
 
-# Timeouts in seconds for thrift calls to beeswax service and the
-# hive metastore. The metastore could talk to an external DB, hence
-# the larger timeout.
-BEESWAX_SERVER_THRIFT_TIMEOUT = 10
-METASTORE_THRIFT_TIMEOUT = 10
-
 def execute_directly(user, query_msg, design=None, notify=False):
   """
   execute_directly(user, query_msg [,design]) -> QueryHistory object
@@ -207,11 +201,12 @@ def db_client():
       res = self._client.get_results_metadata(*args, **kwargs)
       return _decode_struct_attr(res, 'table_dir')
 
+  import ipdb;ipdb.set_trace()
   client = thrift_util.get_client(BeeswaxService.Client,
                                 conf.BEESWAX_SERVER_HOST.get(),
                                 conf.BEESWAX_SERVER_PORT.get(),
                                 service_name="Beeswax (Hive UI) Server",
-                                timeout_seconds=BEESWAX_SERVER_THRIFT_TIMEOUT)
+                                timeout_seconds=conf.BEESWAX_SERVER_CONN_TIMEOUT.get())
   return UnicodeBeeswaxClient(client)
 
 
@@ -306,7 +301,7 @@ def meta_client():
                                 conf.BEESWAX_META_SERVER_HOST.get(),
                                 conf.BEESWAX_META_SERVER_PORT.get(),
                                 service_name="Hive Metadata (Hive UI) Server",
-                                timeout_seconds=METASTORE_THRIFT_TIMEOUT)
+                                timeout_seconds=conf.METASTORE_CONN_TIMEOUT.get())
   return UnicodeMetastoreClient(client)