url_util.py 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. # Copyright (c) 2011-2013 Cloudera, Inc. All rights reserved.
  17. import httplib
  18. import logging
  19. import socket
  20. import sys
  21. import time
  22. import urllib2
  23. from urllib2_kerberos import HTTPKerberosAuthHandler
  24. from M2Crypto import httpslib
  25. from M2Crypto import SSL
  26. from M2Crypto import m2
  27. logging.basicConfig()
  28. LOG = logging.getLogger(__name__)
  29. # urlopen_with_timeout.
  30. #
  31. # The optional secure_http_service_name parameter allows callers to connect to
  32. # secure HTTP servers via the urllib2_kerberos library. We have a modified
  33. # version of the HTTPKerberosAuthHandler code which takes the Kerberos service
  34. # name rather than construct the name using the HTTP request host. We always add
  35. # the HTTPKerberosAuthHandler to urllib2 opener handlers because it has no effect
  36. # if security is not actually enabled.
  37. #
  38. # The optional username and pasword parameters similarly handle setting up HTTP
  39. # digest authentication. Again, this has no effect if HTTP digest authentication
  40. # is not in use on the connection.
  41. #
  42. # The cafile, capath and max_cert_depth control the SSL certificate verification
  43. # behavior. https://www.openssl.org/docs/ssl/SSL_CTX_load_verify_locations.html
  44. # explains the semantics of the parameters. Passing none for both means that
  45. # no verification of the server certification (including the server's hostname)
  46. # will be performed.
  47. def urlopen_with_timeout(url,
  48. data=None,
  49. timeout=None,
  50. secure_http_service_name=None,
  51. username=None,
  52. password=None,
  53. cafile=None,
  54. capath=None,
  55. max_cert_depth=9):
  56. openers = []
  57. openers.append(_make_https_handler(cafile,
  58. capath,
  59. max_cert_depth))
  60. openers.append(HTTPKerberosAuthHandler(secure_http_service_name))
  61. full_url = url
  62. if isinstance(url, urllib2.Request):
  63. full_url = url.get_full_url()
  64. openers.append(_make_http_digest_auth_handler(full_url, username, password))
  65. LOG.info("url_util: urlopen_with_timeout: full_url: %s" % full_url)
  66. if sys.version_info < (2, 6):
  67. # The timeout parameter to urlopen was introduced in Python 2.6.
  68. # To workaround it in older versions of python, we copy, with
  69. # minor modification, httplib.HTTPConnection, and hook it all
  70. # up.
  71. openers.append(_make_timeout_handler(timeout))
  72. opener = urllib2.build_opener(*openers)
  73. LOG.info("url_util: urlopen_with_timeout: sys.version_inf < (2, 6): opener: %s" % opener)
  74. return opener.open(url, data)
  75. else:
  76. openers.append(_make_timeout_handler(timeout))
  77. opener = urllib2.build_opener(*openers)
  78. LOG.info("url_util: urlopen_with_timeout: sys.version_inf > (2, 6): opener: %s" % opener)
  79. return opener.open(url, data, timeout)
  80. def head_request_with_timeout(url,
  81. data=None,
  82. timeout=None,
  83. secure_http_service_name=None,
  84. username=None,
  85. password=None,
  86. cafile=None,
  87. capath=None,
  88. max_cert_depth=9):
  89. class HeadRequest(urllib2.Request):
  90. def get_method(self):
  91. return "HEAD"
  92. if isinstance(url, urllib2.Request):
  93. raise Exception("Unsupported url type: urllib2.Request.")
  94. LOG.info("url_util: head_request_with_timeout: url: %s: timeout: %s" % (url, timeout))
  95. return urlopen_with_timeout(HeadRequest(url),
  96. data,
  97. timeout,
  98. secure_http_service_name,
  99. username,
  100. password,
  101. cafile,
  102. capath,
  103. max_cert_depth)
  104. def _make_timeout_handler(timeout):
  105. # Create these two helper classes fresh each time, since
  106. # timeout needs to be in the closure.
  107. class TimeoutHTTPConnection(httplib.HTTPConnection):
  108. def connect(self):
  109. """Connect to the host and port specified in __init__."""
  110. msg = "getaddrinfo returns an empty list"
  111. for res in socket.getaddrinfo(self.host, self.port, 0,
  112. socket.SOCK_STREAM):
  113. af, socktype, proto, canonname, sa = res
  114. try:
  115. self.sock = socket.socket(af, socktype, proto)
  116. if timeout is not None:
  117. self.sock.settimeout(timeout)
  118. if self.debuglevel > 0:
  119. LOG.info("connect: (%s, %s)" % (self.host, self.port))
  120. self.sock.connect(sa)
  121. except socket.error, msg:
  122. if self.debuglevel > 0:
  123. LOG.info('connect fail:', (self.host, self.port))
  124. if self.sock:
  125. self.sock.close()
  126. self.sock = None
  127. continue
  128. break
  129. if not self.sock:
  130. raise socket.error, msg
  131. class TimeoutHTTPHandler(urllib2.HTTPHandler):
  132. http_request = urllib2.AbstractHTTPHandler.do_request_
  133. def http_open(self, req):
  134. return self.do_open(TimeoutHTTPConnection, req)
  135. return TimeoutHTTPHandler
  136. def _make_http_digest_auth_handler(url, username, password):
  137. password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
  138. password_manager.add_password(None, # realm
  139. url,
  140. username,
  141. password)
  142. return urllib2.HTTPDigestAuthHandler(password_manager)
  143. def _make_https_handler(cafile=None,
  144. capath=None,
  145. max_cert_depth=9):
  146. class HTTPSConnection(httpslib.HTTPSConnection):
  147. """
  148. A class that extends the default HTTPSConnection to ensure two things:
  149. 1) Enforce tlsv1 protocol for all ssl connection. Some older pythons
  150. (e.g., sles11, probably all versions <= 2.6) attempt SSLv23 handshake
  151. that is rejected by newer web servers. See OPSAPS-32192 for an example.
  152. 2) Force validation if cafile/capath is supplied.
  153. """
  154. def __init__(self, host, port=None, **ssl):
  155. # Specifying sslv23 enables the following ssl versions:
  156. # SSLv3, SSLv23, TLSv1, TLSv1.1, and TLSv1.2. We will explicitly exclude
  157. # SSLv3 and SSLv2 below. This mimics what is done by create_default_context
  158. # on newer python versions (python >= 2.7).
  159. ctx = SSL.Context('sslv23')
  160. # SSL_OP_ALL turns on all workarounds for known bugs. See
  161. # https://www.openssl.org/docs/manmaster/ssl/SSL_CTX_set_options.html for
  162. # a full list of these workarounds. I believe that we don't really need
  163. # any of these workarounds, but, this is default in later pythons and is
  164. # future looking.
  165. ctx.set_options(m2.SSL_OP_ALL | m2.SSL_OP_NO_SSLv2 | m2.SSL_OP_NO_SSLv3)
  166. if cafile is not None or capath is not None:
  167. ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert,
  168. max_cert_depth)
  169. ctx.load_verify_info(cafile=cafile, capath=capath)
  170. self._postConnectionCheck = True
  171. else:
  172. ctx.set_verify(SSL.verify_none, max_cert_depth)
  173. self._postConnectionCheck = False
  174. httpslib.HTTPSConnection.__init__(self, host, port, ssl_context=ctx)
  175. def connect(self):
  176. # This is a bit ugly but we need to override the connect method in order
  177. # to disable hostname verification. This is buried deep inside M2Crypto
  178. # and the only way to disable it is to disable post connection checks on
  179. # the socket itself.
  180. self.sock = SSL.Connection(self.ssl_ctx)
  181. if self.session:
  182. self.sock.set_session(self.session)
  183. if not self._postConnectionCheck:
  184. self.sock.postConnectionCheck = None
  185. self.sock.connect((self.host, self.port))
  186. class HTTPSHandler(urllib2.HTTPSHandler):
  187. def https_open(self, req):
  188. return self.do_open(HTTPSConnection, req)
  189. return HTTPSHandler()
  190. def urlopen_with_retry_on_authentication_errors(function,
  191. retries,
  192. sleeptime):
  193. # See OPSAPS-28469: we retry on 401 errors on the presumption that we
  194. # are hitting a race with the kinit from the kt_renewer.
  195. attempt = 1
  196. while True:
  197. try:
  198. return function()
  199. except urllib2.HTTPError, err:
  200. if err.code == 401 and attempt <= retries:
  201. LOG.exception("Autentication error on attempt %d. Retrying after "
  202. "sleeping %f seconds." % (attempt, sleeptime))
  203. time.sleep(sleeptime)
  204. attempt += 1
  205. else:
  206. raise