| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234 |
- # Licensed to Cloudera, Inc. under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. Cloudera, Inc. licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- # Copyright (c) 2011-2013 Cloudera, Inc. All rights reserved.
- import httplib
- import logging
- import socket
- import sys
- import time
- import urllib2
- from urllib2_kerberos import HTTPKerberosAuthHandler
- from M2Crypto import httpslib
- from M2Crypto import SSL
- from M2Crypto import m2
- logging.basicConfig()
- LOG = logging.getLogger(__name__)
- # urlopen_with_timeout.
- #
- # The optional secure_http_service_name parameter allows callers to connect to
- # secure HTTP servers via the urllib2_kerberos library. We have a modified
- # version of the HTTPKerberosAuthHandler code which takes the Kerberos service
- # name rather than construct the name using the HTTP request host. We always add
- # the HTTPKerberosAuthHandler to urllib2 opener handlers because it has no effect
- # if security is not actually enabled.
- #
- # The optional username and pasword parameters similarly handle setting up HTTP
- # digest authentication. Again, this has no effect if HTTP digest authentication
- # is not in use on the connection.
- #
- # The cafile, capath and max_cert_depth control the SSL certificate verification
- # behavior. https://www.openssl.org/docs/ssl/SSL_CTX_load_verify_locations.html
- # explains the semantics of the parameters. Passing none for both means that
- # no verification of the server certification (including the server's hostname)
- # will be performed.
- def urlopen_with_timeout(url,
- data=None,
- timeout=None,
- secure_http_service_name=None,
- username=None,
- password=None,
- cafile=None,
- capath=None,
- max_cert_depth=9):
- openers = []
- openers.append(_make_https_handler(cafile,
- capath,
- max_cert_depth))
- openers.append(HTTPKerberosAuthHandler(secure_http_service_name))
- full_url = url
- if isinstance(url, urllib2.Request):
- full_url = url.get_full_url()
- openers.append(_make_http_digest_auth_handler(full_url, username, password))
- LOG.info("url_util: urlopen_with_timeout: full_url: %s" % full_url)
- if sys.version_info < (2, 6):
- # The timeout parameter to urlopen was introduced in Python 2.6.
- # To workaround it in older versions of python, we copy, with
- # minor modification, httplib.HTTPConnection, and hook it all
- # up.
- openers.append(_make_timeout_handler(timeout))
- opener = urllib2.build_opener(*openers)
- LOG.info("url_util: urlopen_with_timeout: sys.version_inf < (2, 6): opener: %s" % opener)
- return opener.open(url, data)
- else:
- openers.append(_make_timeout_handler(timeout))
- opener = urllib2.build_opener(*openers)
- LOG.info("url_util: urlopen_with_timeout: sys.version_inf > (2, 6): opener: %s" % opener)
- return opener.open(url, data, timeout)
- def head_request_with_timeout(url,
- data=None,
- timeout=None,
- secure_http_service_name=None,
- username=None,
- password=None,
- cafile=None,
- capath=None,
- max_cert_depth=9):
- class HeadRequest(urllib2.Request):
- def get_method(self):
- return "HEAD"
- if isinstance(url, urllib2.Request):
- raise Exception("Unsupported url type: urllib2.Request.")
- LOG.info("url_util: head_request_with_timeout: url: %s: timeout: %s" % (url, timeout))
- return urlopen_with_timeout(HeadRequest(url),
- data,
- timeout,
- secure_http_service_name,
- username,
- password,
- cafile,
- capath,
- max_cert_depth)
- def _make_timeout_handler(timeout):
- # Create these two helper classes fresh each time, since
- # timeout needs to be in the closure.
- class TimeoutHTTPConnection(httplib.HTTPConnection):
- def connect(self):
- """Connect to the host and port specified in __init__."""
- msg = "getaddrinfo returns an empty list"
- for res in socket.getaddrinfo(self.host, self.port, 0,
- socket.SOCK_STREAM):
- af, socktype, proto, canonname, sa = res
- try:
- self.sock = socket.socket(af, socktype, proto)
- if timeout is not None:
- self.sock.settimeout(timeout)
- if self.debuglevel > 0:
- LOG.info("connect: (%s, %s)" % (self.host, self.port))
- self.sock.connect(sa)
- except socket.error, msg:
- if self.debuglevel > 0:
- LOG.info('connect fail:', (self.host, self.port))
- if self.sock:
- self.sock.close()
- self.sock = None
- continue
- break
- if not self.sock:
- raise socket.error, msg
- class TimeoutHTTPHandler(urllib2.HTTPHandler):
- http_request = urllib2.AbstractHTTPHandler.do_request_
- def http_open(self, req):
- return self.do_open(TimeoutHTTPConnection, req)
- return TimeoutHTTPHandler
- def _make_http_digest_auth_handler(url, username, password):
- password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
- password_manager.add_password(None, # realm
- url,
- username,
- password)
- return urllib2.HTTPDigestAuthHandler(password_manager)
- def _make_https_handler(cafile=None,
- capath=None,
- max_cert_depth=9):
- class HTTPSConnection(httpslib.HTTPSConnection):
- """
- A class that extends the default HTTPSConnection to ensure two things:
- 1) Enforce tlsv1 protocol for all ssl connection. Some older pythons
- (e.g., sles11, probably all versions <= 2.6) attempt SSLv23 handshake
- that is rejected by newer web servers. See OPSAPS-32192 for an example.
- 2) Force validation if cafile/capath is supplied.
- """
- def __init__(self, host, port=None, **ssl):
- # Specifying sslv23 enables the following ssl versions:
- # SSLv3, SSLv23, TLSv1, TLSv1.1, and TLSv1.2. We will explicitly exclude
- # SSLv3 and SSLv2 below. This mimics what is done by create_default_context
- # on newer python versions (python >= 2.7).
- ctx = SSL.Context('sslv23')
- # SSL_OP_ALL turns on all workarounds for known bugs. See
- # https://www.openssl.org/docs/manmaster/ssl/SSL_CTX_set_options.html for
- # a full list of these workarounds. I believe that we don't really need
- # any of these workarounds, but, this is default in later pythons and is
- # future looking.
- ctx.set_options(m2.SSL_OP_ALL | m2.SSL_OP_NO_SSLv2 | m2.SSL_OP_NO_SSLv3)
- if cafile is not None or capath is not None:
- ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert,
- max_cert_depth)
- ctx.load_verify_info(cafile=cafile, capath=capath)
- self._postConnectionCheck = True
- else:
- ctx.set_verify(SSL.verify_none, max_cert_depth)
- self._postConnectionCheck = False
- httpslib.HTTPSConnection.__init__(self, host, port, ssl_context=ctx)
- def connect(self):
- # This is a bit ugly but we need to override the connect method in order
- # to disable hostname verification. This is buried deep inside M2Crypto
- # and the only way to disable it is to disable post connection checks on
- # the socket itself.
- self.sock = SSL.Connection(self.ssl_ctx)
- if self.session:
- self.sock.set_session(self.session)
- if not self._postConnectionCheck:
- self.sock.postConnectionCheck = None
- self.sock.connect((self.host, self.port))
- class HTTPSHandler(urllib2.HTTPSHandler):
- def https_open(self, req):
- return self.do_open(HTTPSConnection, req)
- return HTTPSHandler()
- def urlopen_with_retry_on_authentication_errors(function,
- retries,
- sleeptime):
- # See OPSAPS-28469: we retry on 401 errors on the presumption that we
- # are hitting a race with the kinit from the kt_renewer.
- attempt = 1
- while True:
- try:
- return function()
- except urllib2.HTTPError, err:
- if err.code == 401 and attempt <= retries:
- LOG.exception("Autentication error on attempt %d. Retrying after "
- "sleeping %f seconds." % (attempt, sleeptime))
- time.sleep(sleeptime)
- attempt += 1
- else:
- raise
|