conf.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import logging
  18. import os
  19. import socket
  20. import sys
  21. from django.utils.translation import ugettext_lazy as _t, ugettext as _
  22. from desktop.conf import default_ssl_cacerts, default_ssl_validate, AUTH_USERNAME as DEFAULT_AUTH_USERNAME, \
  23. AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD, has_connectors
  24. from desktop.lib.conf import ConfigSection, Config, coerce_bool, coerce_csv, coerce_password_from_script
  25. from desktop.lib.exceptions import StructuredThriftTransportException
  26. from desktop.lib.paths import get_desktop_root
  27. from impala.impala_flags import get_max_result_cache_size, is_impersonation_enabled, is_kerberos_enabled, is_webserver_spnego_enabled
  28. from impala.settings import NICE_NAME
  29. LOG = logging.getLogger(__name__)
  30. SERVER_HOST = Config(
  31. key="server_host",
  32. help=_t("Host of the Impala Server."),
  33. default="localhost")
  34. SERVER_PORT = Config(
  35. key="server_port",
  36. help=_t("Port of the Impala Server."),
  37. default=21050,
  38. type=int)
  39. COORDINATOR_URL = Config(
  40. key="coordinator_url",
  41. help=_t("URL of the Impala Coordinator Server."),
  42. type=str,
  43. default="")
  44. IMPALA_PRINCIPAL=Config(
  45. key='impala_principal',
  46. help=_t("Kerberos principal name for Impala. Typically 'impala/hostname.foo.com'."),
  47. type=str,
  48. default="impala/%s" % socket.getfqdn())
  49. IMPERSONATION_ENABLED=Config(
  50. key='impersonation_enabled',
  51. help=_t("Turn on/off impersonation mechanism when talking to Impala."),
  52. type=coerce_bool,
  53. dynamic_default=is_impersonation_enabled)
  54. QUERYCACHE_ROWS=Config(
  55. key='querycache_rows',
  56. help=_t("Number of initial rows of a resultset to ask Impala to cache in order to"
  57. " support re-fetching them for downloading them."
  58. " Set to 0 for disabling the option and backward compatibility."),
  59. type=int,
  60. dynamic_default=get_max_result_cache_size)
  61. SERVER_CONN_TIMEOUT = Config(
  62. key='server_conn_timeout',
  63. default=120,
  64. type=int,
  65. help=_t('Timeout in seconds for Thrift calls.'))
  66. CLOSE_QUERIES = Config(
  67. key="close_queries",
  68. help=_t("Hue will try to close the Impala query when the user leaves the editor page. "
  69. "This will free all the query resources in Impala, but also make its results inaccessible."),
  70. type=coerce_bool,
  71. default=True
  72. )
  73. QUERY_TIMEOUT_S = Config(
  74. key="query_timeout_s",
  75. help=_t("If QUERY_TIMEOUT_S > 0, the query will be timed out (i.e. cancelled) if Impala does not do any work"
  76. " (compute or send back results) for that query within QUERY_TIMEOUT_S seconds."),
  77. type=int,
  78. default=300
  79. )
  80. SESSION_TIMEOUT_S = Config(
  81. key="session_timeout_s",
  82. help=_t("If SESSION_TIMEOUT_S > 0, the session will be timed out (i.e. cancelled) if Impala does not do any work"
  83. " (compute or send back results) for that session within SESSION_TIMEOUT_S seconds. Default: 15 min."),
  84. type=int,
  85. default=15 * 60
  86. )
  87. CONFIG_WHITELIST = Config(
  88. key='config_whitelist',
  89. default='debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s,request_pool',
  90. type=coerce_csv,
  91. help=_t('A comma-separated list of white-listed Impala configuration properties that users are authorized to set.')
  92. )
  93. IMPALA_CONF_DIR = Config(
  94. key='impala_conf_dir',
  95. help=_t('Impala configuration directory, where impala_flags is located.'),
  96. type=str,
  97. default=os.environ.get("HUE_CONF_DIR", get_desktop_root("conf")) + '/impala-conf'
  98. )
  99. SSL = ConfigSection(
  100. key='ssl',
  101. help=_t('SSL configuration for the server.'),
  102. members=dict(
  103. ENABLED = Config(
  104. key="enabled",
  105. help=_t("SSL communication enabled for this server."),
  106. type=coerce_bool,
  107. default=False
  108. ),
  109. CACERTS = Config(
  110. key="cacerts",
  111. help=_t("Path to Certificate Authority certificates."),
  112. type=str,
  113. dynamic_default=default_ssl_cacerts,
  114. ),
  115. KEY = Config(
  116. key="key",
  117. help=_t("Path to the private key file, e.g. /etc/hue/key.pem"),
  118. type=str,
  119. default=None
  120. ),
  121. CERT = Config(
  122. key="cert",
  123. help=_t("Path to the public certificate file, e.g. /etc/hue/cert.pem"),
  124. type=str,
  125. default=None
  126. ),
  127. VALIDATE = Config(
  128. key="validate",
  129. help=_t("Choose whether Hue should validate certificates received from the server."),
  130. type=coerce_bool,
  131. dynamic_default=default_ssl_validate,
  132. )
  133. )
  134. )
  135. def get_auth_username():
  136. """Get from top level default from desktop"""
  137. return DEFAULT_AUTH_USERNAME.get()
  138. AUTH_USERNAME = Config(
  139. key="auth_username",
  140. help=_t("Auth username of the hue user used for authentications."),
  141. private=True,
  142. dynamic_default=get_auth_username)
  143. def get_auth_password():
  144. """Get from script or backward compatibility"""
  145. password = AUTH_PASSWORD_SCRIPT.get()
  146. if password:
  147. return password
  148. return DEFAULT_AUTH_PASSWORD.get()
  149. AUTH_PASSWORD = Config(
  150. key="auth_password",
  151. help=_t("LDAP/PAM/.. password of the hue user used for authentications."),
  152. private=True,
  153. dynamic_default=get_auth_password
  154. )
  155. AUTH_PASSWORD_SCRIPT = Config(
  156. key="auth_password_script",
  157. help=_t("Execute this script to produce the auth password. This will be used when `auth_password` is not set."),
  158. private=True,
  159. type=coerce_password_from_script,
  160. default=None
  161. )
  162. def get_daemon_config(key):
  163. from metadata.conf import MANAGER
  164. from metadata.manager_client import ManagerApi
  165. if MANAGER.API_URL.get():
  166. return ManagerApi().get_impalad_config(key=key, impalad_host=SERVER_HOST.get())
  167. return None
  168. def get_daemon_api_username():
  169. """
  170. Try to get daemon_api_username from Cloudera Manager API
  171. """
  172. return get_daemon_config('webserver_htpassword_user')
  173. def get_daemon_api_password():
  174. """
  175. Try to get daemon_api_password from Cloudera Manager API
  176. """
  177. return get_daemon_config('webserver_htpassword_password')
  178. DAEMON_API_PASSWORD = Config(
  179. key="daemon_api_password",
  180. help=_t("Password for Impala Daemon when username/password authentication is enabled for the Impala Daemon UI."),
  181. private=True,
  182. dynamic_default=get_daemon_api_password
  183. )
  184. DAEMON_API_PASSWORD_SCRIPT = Config(
  185. key="daemon_api_password_script",
  186. help=_t("Execute this script to produce the Impala Daemon Password. This will be used when `daemon_api_password` is not set."),
  187. private=True,
  188. type=coerce_password_from_script,
  189. default=None
  190. )
  191. DAEMON_API_USERNAME = Config(
  192. key="daemon_api_username",
  193. help=_t("Username for Impala Daemon when username/password authentication is enabled for the Impala Daemon UI."),
  194. private=True,
  195. dynamic_default=get_daemon_api_username
  196. )
  197. def get_use_sasl_default():
  198. """kerberos enabled or password is specified"""
  199. return is_kerberos_enabled() or AUTH_PASSWORD.get() is not None # Maps closely to legacy behavior
  200. USE_SASL = Config(
  201. key="use_sasl",
  202. help=_t("Use SASL framework to establish connection to host."),
  203. private=False,
  204. type=coerce_bool,
  205. dynamic_default=get_use_sasl_default
  206. )
  207. def config_validator(user):
  208. # dbms is dependent on beeswax.conf, import in method to avoid circular dependency
  209. from beeswax.design import hql_query
  210. from beeswax.server import dbms
  211. from beeswax.server.dbms import get_query_server_config
  212. res = []
  213. if has_connectors():
  214. return res
  215. try:
  216. try:
  217. if not 'test' in sys.argv: # Avoid tests hanging
  218. query_server = get_query_server_config(name='impala')
  219. server = dbms.get(user, query_server)
  220. query = hql_query("SELECT 'Hello World!';")
  221. handle = server.execute_and_wait(query, timeout_sec=10.0)
  222. if handle:
  223. server.fetch(handle, rows=100)
  224. server.close(handle)
  225. except StructuredThriftTransportException as ex:
  226. if 'TSocket read 0 bytes' in str(ex): # this message appears when authentication fails
  227. msg = "Failed to authenticate to Impalad, check authentication configurations."
  228. LOG.exception(msg)
  229. res.append((NICE_NAME, _(msg)))
  230. else:
  231. raise ex
  232. except Exception as ex:
  233. msg = "No available Impalad to send queries to."
  234. LOG.exception(msg)
  235. res.append((NICE_NAME, _(msg)))
  236. return res