ソースを参照

HUE-9062 [beeswax] Add http thrift port configuration

Change-Id: I341d0d93d301383c76cb6eac70e414dd3260c3f7
Jean-Francois Desjeans Gauthier 6 年 前
コミット
6368aef66b

+ 23 - 7
apps/beeswax/src/beeswax/conf.py

@@ -72,14 +72,14 @@ CACHE_TIMEOUT = Config(
 
 LLAP_SERVER_PORT = Config(
   key="llap_server_port",
-  help=_t("Configure the base port the Hive Server Interactive runs on (10500 default)."),
+  help=_t("LLAP binary Thrift port (10500 default)."),
   default=10500,
   type=int
 )
 
 LLAP_SERVER_THRIFT_PORT = Config(
   key="llap_server_thrift_port",
-  help=_t("Configure the thrift port the Hive Server Interactive runs on (10501 default)"),
+  help=_t("LLAP http Thrift port (10501 default)"),
   default=10501,
   type=int
 )
@@ -97,10 +97,26 @@ HIVE_SERVER_HOST = Config(
          "the fully-qualified domain name (FQDN) is required"),
   default="localhost")
 
+def get_hive_thrift_binary_port():
+  """Devise port from core-site Thrift / execution mode & Http port"""
+  from beeswax.hive_site import hiveserver2_thrift_binary_port, get_hive_execution_mode   # Cyclic dependency
+  return hiveserver2_thrift_binary_port() or (10500 if (get_hive_execution_mode() or '').lower() == 'llap' else 10000)
+
 HIVE_SERVER_PORT = Config(
   key="hive_server_port",
-  help=_t("Configure the port the HiveServer2 server runs on."),
-  default=10000,
+  help=_t("Configure the binary Thrift port for HiveServer2."),
+  dynamic_default=get_hive_thrift_binary_port,
+  type=int)
+
+def get_hive_thrift_http_port():
+  """Devise port from core-site Thrift / execution mode & Http port"""
+  from beeswax.hive_site import hiveserver2_thrift_http_port, get_hive_execution_mode   # Cyclic dependency
+  return hiveserver2_thrift_http_port() or (10501 if (get_hive_execution_mode() or '').lower() == 'llap'  else 10001)
+
+HIVE_HTTP_THRIFT_PORT = Config(
+  key="hive_server_http_port",
+  help=_t("Configure the Http Thrift port for HiveServer2."),
+  dynamic_default=get_hive_thrift_http_port,
   type=int)
 
 HIVE_METASTORE_HOST = Config(
@@ -141,8 +157,8 @@ USE_GET_LOG_API = Config( # To remove in Hue 4
   key='use_get_log_api',
   default=False,
   type=coerce_bool,
-  help=_t('Choose whether to use the old GetLog() thrift call from before Hive 0.14 to retrieve the logs.'
-          'If false, use the FetchResults() thrift call from Hive 1.0 or more instead.')
+  help=_t('Choose whether to use the old GetLog() Thrift call from before Hive 0.14 to retrieve the logs.'
+          'If false, use the FetchResults() Thrift call from Hive 1.0 or more instead.')
 )
 
 BROWSE_PARTITIONED_TABLE_LIMIT = Config( # Deprecated, to remove in Hue 4
@@ -307,4 +323,4 @@ USE_SASL = Config(
   help=_t("Use SASL framework to establish connection to host"),
   private=False,
   type=coerce_bool,
-  dynamic_default=get_use_sasl_default)
+  dynamic_default=get_use_sasl_default)

+ 10 - 2
apps/beeswax/src/beeswax/hive_site.py

@@ -58,6 +58,7 @@ _CNF_HIVESERVER2_TRUSTSTORE_PATH = 'hive.server2.truststore.path'
 _CNF_HIVESERVER2_TRUSTSTORE_PASSWORD = 'hive.server2.truststore.password'
 
 _CNF_HIVESERVER2_TRANSPORT_MODE = 'hive.server2.transport.mode'
+_CNF_HIVESERVER2_THRIFT_BINARY_PORT = 'hive.server2.thrift.port'
 _CNF_HIVESERVER2_THRIFT_HTTP_PORT = 'hive.server2.thrift.http.port'
 _CNF_HIVESERVER2_THRIFT_HTTP_PATH = 'hive.server2.thrift.http.path'
 _CNF_HIVESERVER2_THRIFT_SASL_QOP = 'hive.server2.thrift.sasl.qop'
@@ -66,6 +67,7 @@ _CNF_HIVESERVER2_USE_SASL = 'hive.metastore.sasl.enabled'
 
 _CNF_HIVE_SUPPORT_CONCURRENCY = 'hive.support.concurrency'
 _CNF_HIVE_HOOK_PROTO_BASE_DIR = 'hive.hook.proto.base-directory'
+_CNF_HIVE_EXECUTION_MODE = 'hive.execution.mode'
 
 
 # Host is whatever up to the colon. Allow and ignore a trailing slash.
@@ -149,7 +151,7 @@ def hiveserver2_impersonation_enabled():
 
 def hiveserver2_jdbc_url():
   is_transport_mode_http = hiveserver2_transport_mode() == 'HTTP'
-  urlbase = 'jdbc:hive2://%s:%s/default' % (beeswax.conf.HIVE_SERVER_HOST.get(), hiveserver2_thrift_http_port() if is_transport_mode_http else beeswax.conf.HIVE_SERVER_PORT.get())
+  urlbase = 'jdbc:hive2://%s:%s/default' % (beeswax.conf.HIVE_SERVER_HOST.get(), beeswax.conf.HIVE_HTTP_THRIFT_PORT.get() if is_transport_mode_http else beeswax.conf.HIVE_SERVER_PORT.get())
 
   if get_conf().get(_CNF_HIVESERVER2_USE_SSL, 'FALSE').upper() == 'TRUE':
     urlbase += ';ssl=true'
@@ -177,8 +179,11 @@ def hiveserver2_use_ssl():
 def hiveserver2_transport_mode():
   return get_conf().get(_CNF_HIVESERVER2_TRANSPORT_MODE, 'TCP').upper()
 
+def hiveserver2_thrift_binary_port():
+  return get_conf().get(_CNF_HIVESERVER2_THRIFT_BINARY_PORT)
+
 def hiveserver2_thrift_http_port():
-  return get_conf().get(_CNF_HIVESERVER2_THRIFT_HTTP_PORT, '10001')
+  return get_conf().get(_CNF_HIVESERVER2_THRIFT_HTTP_PORT)
 
 def hiveserver2_thrift_http_path():
   return get_conf().get(_CNF_HIVESERVER2_THRIFT_HTTP_PATH, 'cliservice')
@@ -195,6 +200,9 @@ def has_concurrency_support():
 def get_hive_hook_proto_base_directory():
   return get_conf().get(_CNF_HIVE_HOOK_PROTO_BASE_DIR)
 
+def get_hive_execution_mode():
+  return get_conf().get(_CNF_HIVE_EXECUTION_MODE)
+
 def _parse_hive_site():
   """
   Parse hive-site.xml and store in _HIVE_SITE_DICT

+ 4 - 4
apps/beeswax/src/beeswax/server/dbms.py

@@ -39,7 +39,7 @@ from indexer.file_format import HiveFormat
 from libzookeeper import conf as libzookeeper_conf
 
 from beeswax import hive_site
-from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_METASTORE_HOST, HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
+from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_SERVER_HOST, HIVE_HTTP_THRIFT_PORT, HIVE_METASTORE_HOST, HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
   AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
   HIVE_DISCOVERY_HS2, HIVE_DISCOVERY_LLAP, HIVE_DISCOVERY_LLAP_HA, HIVE_DISCOVERY_LLAP_ZNODE, CACHE_TIMEOUT, \
   LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT, USE_SASL as HIVE_USE_SASL
@@ -120,7 +120,7 @@ def get_query_server_config(name='beeswax', connector=None):
                   cache.set("llap", json.dumps({"host": llap_servers["addresses"][0]["host"], "port": llap_servers["addresses"][0]["port"]}), CACHE_TIMEOUT.get())
             else:
               LOG.error("LLAP Endpoint not found, reverting to HiveServer2")
-              cache.set("llap", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": hive_site.hiveserver2_thrift_http_port()}), CACHE_TIMEOUT.get())
+              cache.set("llap", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}), CACHE_TIMEOUT.get())
           else:
             znode = "{0}".format(HIVE_DISCOVERY_LLAP_ZNODE.get())
             LOG.debug("Setting up LLAP with the following node {0}".format(znode))
@@ -146,10 +146,10 @@ def get_query_server_config(name='beeswax', connector=None):
             server_to_use = 0 # if CONF.HIVE_SPREAD.get() randint(0, len(hiveservers)-1) else 0
             cache.set("hiveserver2", json.dumps({"host": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[0], "port": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[1]}))
           else:
-            cache.set("hiveserver2", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": hive_site.hiveserver2_thrift_http_port()}))
+            cache.set("hiveserver2", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}))
           zk.stop()
         else:
-          cache.set("hiveserver2", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": hive_site.hiveserver2_thrift_http_port()}))
+          cache.set("hiveserver2", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}))
       activeEndpoint = json.loads(cache.get("hiveserver2"))
 
     if name == 'impala':

+ 6 - 3
desktop/conf.dist/hue.ini

@@ -1164,16 +1164,19 @@
   # If Kerberos security is enabled, use fully-qualified domain name (FQDN).
   ## hive_server_host=localhost
 
-  # Port where HiveServer2 Thrift server runs on.
+  # Binary thrift port for HiveServer2.
   ## hive_server_port=10000
 
+  # Http thrift port for HiveServer2.
+  ## hive_server_http_port=10001
+
   # Host where LLAP is running
   ## llap_server_host = localhost
 
-  # Main Port where LLAP runs
+  # LLAP binary thrift port
   ## llap_server_port = 10500
 
-  # Port where LLAP Thrift server runs on
+  # LLAP HTTP Thrift port
   ## llap_server_thrift_port = 10501
 
   # Alternatively, use Service Discovery for LLAP (Hive Server Interactive) and/or Hiveserver2, this will override server and thrift port

+ 5 - 2
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1151,13 +1151,16 @@
   # Port where HiveServer2 Thrift server runs on.
   ## hive_server_port=10000
 
+  # Http thrift port for HiveServer2.
+  ## hive_server_http_port=10001
+
   # Host where LLAP is running
   ## llap_server_host = localhost
 
-  # Main Port where LLAP runs
+  # LLAP binary thrift port
   ## llap_server_port = 10500
 
-  # Port where LLAP Thrift server runs on
+  # LLAP HTTP Thrift port
   ## llap_server_thrift_port = 10501
 
   # Alternatively, use Service Discovery for LLAP (Hive Server Interactive) and/or Hiveserver2, this will override server and thrift port