Browse Source

Adding connection timeout for zookeeper connection to get hive znode (#3212)

If incase ZK is slow to respond back, the default timeout is 15 seconds may not be ideal for busy cluster where the zookeeper connection can be established after 30 seconds or so.
Hence set the default timeout to 30 seconds for the ZK connection.

Jira: CDPD-50059

Ref: https://kazoo.readthedocs.io/en/latest/_modules/kazoo/client.html#KazooClient.start
Mahesh Balakrishnan 2 years ago
parent
commit
d3df801115

+ 6 - 0
apps/beeswax/src/beeswax/conf.py

@@ -159,6 +159,12 @@ SERVER_CONN_TIMEOUT = Config(
   type=int,
   help=_t('Timeout in seconds for Thrift calls.'))
 
+ZOOKEEPER_CONN_TIMEOUT = Config(
+  key='zookeeper_conn_timeout',
+  default=30,
+  type=int,
+  help=_t('Timeout in seconds for zookeeper connection.'))
+
 USE_GET_LOG_API = Config( # To remove in Hue 4
   key='use_get_log_api',
   default=False,

+ 3 - 3
apps/beeswax/src/beeswax/server/dbms.py

@@ -39,7 +39,7 @@ from libzookeeper import conf as libzookeeper_conf
 
 from azure.abfs import abfspath
 from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_SERVER_HOST, HIVE_HTTP_THRIFT_PORT, HIVE_METASTORE_HOST, \
-    HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
+    HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, ZOOKEEPER_CONN_TIMEOUT, \
     AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
     HIVE_DISCOVERY_HS2, HIVE_DISCOVERY_LLAP, HIVE_DISCOVERY_LLAP_HA, HIVE_DISCOVERY_LLAP_ZNODE, CACHE_TIMEOUT, \
     LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT, USE_SASL as HIVE_USE_SASL, CLOSE_SESSIONS, has_session_pool, \
@@ -83,7 +83,7 @@ reset_ha()
 def get_zk_hs2():
   hiveservers = None
   zk = KazooClient(hosts=libzookeeper_conf.ENSEMBLE.get(), read_only=True)
-  zk.start()
+  zk.start(timeout=ZOOKEEPER_CONN_TIMEOUT.get())
   znode = HIVE_DISCOVERY_HIVESERVER2_ZNODE.get()
   if zk.exists(znode):
     LOG.debug("Selecting up Hive server via the following node {0}".format(znode))
@@ -153,7 +153,7 @@ def get_query_server_config(name='beeswax', connector=None):
         if HIVE_DISCOVERY_LLAP.get():
           LOG.debug("Checking zookeeper for discovering Hive LLAP server endpoint")
           zk = KazooClient(hosts=libzookeeper_conf.ENSEMBLE.get(), read_only=True)
-          zk.start()
+          zk.start(timeout=ZOOKEEPER_CONN_TIMEOUT.get())
           if HIVE_DISCOVERY_LLAP_HA.get():
             znode = "{0}/instances".format(HIVE_DISCOVERY_LLAP_ZNODE.get())
             LOG.debug("Setting up Hive LLAP HA with the following node {0}".format(znode))

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1345,6 +1345,9 @@ submit_to=True
 # Timeout in seconds for thrift calls to Hive service
 ## server_conn_timeout=120
 
+# Timeout in seconds for zookeeper connectivity to get the znode information
+## zookeeper_conn_timeout=30
+
 # Choose whether to use the old GetLog() thrift call from before Hive 0.14 to retrieve the logs.
 # If false, use the FetchResults() thrift call from Hive 1.0 or more instead.
 ## use_get_log_api=false

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1328,6 +1328,9 @@
   # Timeout in seconds for thrift calls to Hive service
   ## server_conn_timeout=120
 
+  # Timeout in seconds for zookeeper connectivity to get the znode information
+  ## zookeeper_conn_timeout=30
+
   # Choose whether to use the old GetLog() thrift call from before Hive 0.14 to retrieve the logs.
   # If false, use the FetchResults() thrift call from Hive 1.0 or more instead.
   ## use_get_log_api=false