279 lines
10 KiB
Python
279 lines
10 KiB
Python
|
import os.path
|
||
|
|
import logging
|
||
|
|
import socket
|
||
|
|
from base64 import b64encode
|
||
|
|
|
||
|
|
from urllib3 import PoolManager, ProxyManager, proxy_from_url, Timeout
|
||
|
|
from urllib3.util.ssl_ import (
|
||
|
|
ssl, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, DEFAULT_CIPHERS,
|
||
|
|
)
|
||
|
|
from urllib3.exceptions import SSLError as URLLib3SSLError
|
||
|
|
from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError
|
||
|
|
from urllib3.exceptions import ConnectTimeoutError as URLLib3ConnectTimeoutError
|
||
|
|
from urllib3.exceptions import NewConnectionError, ProtocolError, ProxyError
|
||
|
|
try:
|
||
|
|
# Always import the original SSLContext, even if it has been patched
|
||
|
|
from urllib3.contrib.pyopenssl import orig_util_SSLContext as SSLContext
|
||
|
|
except ImportError:
|
||
|
|
from urllib3.util.ssl_ import SSLContext
|
||
|
|
|
||
|
|
import botocore.awsrequest
|
||
|
|
from botocore.vendored import six
|
||
|
|
from botocore.vendored.six.moves.urllib_parse import unquote
|
||
|
|
from botocore.compat import filter_ssl_warnings, urlparse
|
||
|
|
from botocore.exceptions import (
|
||
|
|
ConnectionClosedError, EndpointConnectionError, HTTPClientError,
|
||
|
|
ReadTimeoutError, ProxyConnectionError, ConnectTimeoutError, SSLError
|
||
|
|
)
|
||
|
|
|
||
|
|
filter_ssl_warnings()
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
DEFAULT_TIMEOUT = 60
|
||
|
|
MAX_POOL_CONNECTIONS = 10
|
||
|
|
DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem')
|
||
|
|
|
||
|
|
try:
|
||
|
|
from certifi import where
|
||
|
|
except ImportError:
|
||
|
|
def where():
|
||
|
|
return DEFAULT_CA_BUNDLE
|
||
|
|
|
||
|
|
|
||
|
|
def get_cert_path(verify):
|
||
|
|
if verify is not True:
|
||
|
|
return verify
|
||
|
|
|
||
|
|
return where()
|
||
|
|
|
||
|
|
|
||
|
|
def create_urllib3_context(ssl_version=None, cert_reqs=None,
|
||
|
|
options=None, ciphers=None):
|
||
|
|
""" This function is a vendored version of the same function in urllib3
|
||
|
|
|
||
|
|
We vendor this function to ensure that the SSL contexts we construct
|
||
|
|
always use the std lib SSLContext instead of pyopenssl.
|
||
|
|
"""
|
||
|
|
context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)
|
||
|
|
|
||
|
|
# Setting the default here, as we may have no ssl module on import
|
||
|
|
cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
|
||
|
|
|
||
|
|
if options is None:
|
||
|
|
options = 0
|
||
|
|
# SSLv2 is easily broken and is considered harmful and dangerous
|
||
|
|
options |= OP_NO_SSLv2
|
||
|
|
# SSLv3 has several problems and is now dangerous
|
||
|
|
options |= OP_NO_SSLv3
|
||
|
|
# Disable compression to prevent CRIME attacks for OpenSSL 1.0+
|
||
|
|
# (issue urllib3#309)
|
||
|
|
options |= OP_NO_COMPRESSION
|
||
|
|
|
||
|
|
context.options |= options
|
||
|
|
|
||
|
|
if getattr(context, 'supports_set_ciphers', True):
|
||
|
|
# Platform-specific: Python 2.6
|
||
|
|
context.set_ciphers(ciphers or DEFAULT_CIPHERS)
|
||
|
|
|
||
|
|
context.verify_mode = cert_reqs
|
||
|
|
if getattr(context, 'check_hostname', None) is not None:
|
||
|
|
# Platform-specific: Python 3.2
|
||
|
|
# We do our own verification, including fingerprints and alternative
|
||
|
|
# hostnames. So disable it here
|
||
|
|
context.check_hostname = False
|
||
|
|
return context
|
||
|
|
|
||
|
|
|
||
|
|
class ProxyConfiguration(object):
|
||
|
|
"""Represents a proxy configuration dictionary.
|
||
|
|
|
||
|
|
This class represents a proxy configuration dictionary and provides utility
|
||
|
|
functions to retreive well structured proxy urls and proxy headers from the
|
||
|
|
proxy configuration dictionary.
|
||
|
|
"""
|
||
|
|
def __init__(self, proxies=None):
|
||
|
|
if proxies is None:
|
||
|
|
proxies = {}
|
||
|
|
self._proxies = proxies
|
||
|
|
|
||
|
|
def proxy_url_for(self, url):
|
||
|
|
"""Retrirves the corresponding proxy url for a given url. """
|
||
|
|
parsed_url = urlparse(url)
|
||
|
|
proxy = self._proxies.get(parsed_url.scheme)
|
||
|
|
if proxy:
|
||
|
|
proxy = self._fix_proxy_url(proxy)
|
||
|
|
return proxy
|
||
|
|
|
||
|
|
def proxy_headers_for(self, proxy_url):
|
||
|
|
"""Retrirves the corresponding proxy headers for a given proxy url. """
|
||
|
|
headers = {}
|
||
|
|
username, password = self._get_auth_from_url(proxy_url)
|
||
|
|
if username and password:
|
||
|
|
basic_auth = self._construct_basic_auth(username, password)
|
||
|
|
headers['Proxy-Authorization'] = basic_auth
|
||
|
|
return headers
|
||
|
|
|
||
|
|
def _fix_proxy_url(self, proxy_url):
|
||
|
|
if proxy_url.startswith('http:') or proxy_url.startswith('https:'):
|
||
|
|
return proxy_url
|
||
|
|
elif proxy_url.startswith('//'):
|
||
|
|
return 'http:' + proxy_url
|
||
|
|
else:
|
||
|
|
return 'http://' + proxy_url
|
||
|
|
|
||
|
|
def _construct_basic_auth(self, username, password):
|
||
|
|
auth_str = '{0}:{1}'.format(username, password)
|
||
|
|
encoded_str = b64encode(auth_str.encode('ascii')).strip().decode()
|
||
|
|
return 'Basic {0}'.format(encoded_str)
|
||
|
|
|
||
|
|
def _get_auth_from_url(self, url):
|
||
|
|
parsed_url = urlparse(url)
|
||
|
|
try:
|
||
|
|
return unquote(parsed_url.username), unquote(parsed_url.password)
|
||
|
|
except (AttributeError, TypeError):
|
||
|
|
return None, None
|
||
|
|
|
||
|
|
|
||
|
|
class URLLib3Session(object):
|
||
|
|
"""A basic HTTP client that supports connection pooling and proxies.
|
||
|
|
|
||
|
|
This class is inspired by requests.adapters.HTTPAdapter, but has been
|
||
|
|
boiled down to meet the use cases needed by botocore. For the most part
|
||
|
|
this classes matches the functionality of HTTPAdapter in requests v2.7.0
|
||
|
|
(the same as our vendored version). The only major difference of note is
|
||
|
|
that we currently do not support sending chunked requests. While requests
|
||
|
|
v2.7.0 implemented this themselves, later version urllib3 support this
|
||
|
|
directly via a flag to urlopen so enabling it if needed should be trivial.
|
||
|
|
"""
|
||
|
|
def __init__(self,
|
||
|
|
verify=True,
|
||
|
|
proxies=None,
|
||
|
|
timeout=None,
|
||
|
|
max_pool_connections=MAX_POOL_CONNECTIONS,
|
||
|
|
):
|
||
|
|
self._verify = verify
|
||
|
|
self._proxy_config = ProxyConfiguration(proxies=proxies)
|
||
|
|
self._pool_classes_by_scheme = {
|
||
|
|
'http': botocore.awsrequest.AWSHTTPConnectionPool,
|
||
|
|
'https': botocore.awsrequest.AWSHTTPSConnectionPool,
|
||
|
|
}
|
||
|
|
if timeout is None:
|
||
|
|
timeout = DEFAULT_TIMEOUT
|
||
|
|
if not isinstance(timeout, (int, float)):
|
||
|
|
timeout = Timeout(connect=timeout[0], read=timeout[1])
|
||
|
|
self._timeout = timeout
|
||
|
|
self._max_pool_connections = max_pool_connections
|
||
|
|
self._proxy_managers = {}
|
||
|
|
self._manager = PoolManager(
|
||
|
|
strict=True,
|
||
|
|
timeout=self._timeout,
|
||
|
|
maxsize=self._max_pool_connections,
|
||
|
|
ssl_context=self._get_ssl_context(),
|
||
|
|
)
|
||
|
|
self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme
|
||
|
|
|
||
|
|
def _get_ssl_context(self):
|
||
|
|
return create_urllib3_context()
|
||
|
|
|
||
|
|
def _get_proxy_manager(self, proxy_url):
|
||
|
|
if proxy_url not in self._proxy_managers:
|
||
|
|
proxy_headers = self._proxy_config.proxy_headers_for(proxy_url)
|
||
|
|
proxy_manager = proxy_from_url(
|
||
|
|
proxy_url,
|
||
|
|
strict=True,
|
||
|
|
timeout=self._timeout,
|
||
|
|
proxy_headers=proxy_headers,
|
||
|
|
maxsize=self._max_pool_connections,
|
||
|
|
ssl_context=self._get_ssl_context(),
|
||
|
|
)
|
||
|
|
proxy_manager.pool_classes_by_scheme = self._pool_classes_by_scheme
|
||
|
|
self._proxy_managers[proxy_url] = proxy_manager
|
||
|
|
|
||
|
|
return self._proxy_managers[proxy_url]
|
||
|
|
|
||
|
|
def _path_url(self, url):
|
||
|
|
parsed_url = urlparse(url)
|
||
|
|
path = parsed_url.path
|
||
|
|
if not path:
|
||
|
|
path = '/'
|
||
|
|
if parsed_url.query:
|
||
|
|
path = path + '?' + parsed_url.query
|
||
|
|
return path
|
||
|
|
|
||
|
|
def _setup_ssl_cert(self, conn, url, verify):
|
||
|
|
if url.lower().startswith('https') and verify:
|
||
|
|
conn.cert_reqs = 'CERT_REQUIRED'
|
||
|
|
conn.ca_certs = get_cert_path(verify)
|
||
|
|
else:
|
||
|
|
conn.cert_reqs = 'CERT_NONE'
|
||
|
|
conn.ca_certs = None
|
||
|
|
|
||
|
|
def _get_connection_manager(self, url, proxy_url=None):
|
||
|
|
if proxy_url:
|
||
|
|
manager = self._get_proxy_manager(proxy_url)
|
||
|
|
else:
|
||
|
|
manager = self._manager
|
||
|
|
return manager
|
||
|
|
|
||
|
|
def _get_request_target(self, url, proxy_url):
|
||
|
|
if proxy_url and url.startswith('http:'):
|
||
|
|
# HTTP proxies expect the request_target to be the absolute url to
|
||
|
|
# know which host to establish a connection to
|
||
|
|
return url
|
||
|
|
else:
|
||
|
|
# otherwise just set the request target to the url path
|
||
|
|
return self._path_url(url)
|
||
|
|
|
||
|
|
def send(self, request):
|
||
|
|
try:
|
||
|
|
proxy_url = self._proxy_config.proxy_url_for(request.url)
|
||
|
|
manager = self._get_connection_manager(request.url, proxy_url)
|
||
|
|
conn = manager.connection_from_url(request.url)
|
||
|
|
self._setup_ssl_cert(conn, request.url, self._verify)
|
||
|
|
|
||
|
|
request_target = self._get_request_target(request.url, proxy_url)
|
||
|
|
urllib_response = conn.urlopen(
|
||
|
|
method=request.method,
|
||
|
|
url=request_target,
|
||
|
|
body=request.body,
|
||
|
|
headers=request.headers,
|
||
|
|
retries=False,
|
||
|
|
assert_same_host=False,
|
||
|
|
preload_content=False,
|
||
|
|
decode_content=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
http_response = botocore.awsrequest.AWSResponse(
|
||
|
|
request.url,
|
||
|
|
urllib_response.status,
|
||
|
|
urllib_response.headers,
|
||
|
|
urllib_response,
|
||
|
|
)
|
||
|
|
|
||
|
|
if not request.stream_output:
|
||
|
|
# Cause the raw stream to be exhausted immediately. We do it
|
||
|
|
# this way instead of using preload_content because
|
||
|
|
# preload_content will never buffer chunked responses
|
||
|
|
http_response.content
|
||
|
|
|
||
|
|
return http_response
|
||
|
|
except URLLib3SSLError as e:
|
||
|
|
raise SSLError(endpoint_url=request.url, error=e)
|
||
|
|
except (NewConnectionError, socket.gaierror) as e:
|
||
|
|
raise EndpointConnectionError(endpoint_url=request.url, error=e)
|
||
|
|
except ProxyError as e:
|
||
|
|
raise ProxyConnectionError(proxy_url=proxy_url, error=e)
|
||
|
|
except URLLib3ConnectTimeoutError as e:
|
||
|
|
raise ConnectTimeoutError(endpoint_url=request.url, error=e)
|
||
|
|
except URLLib3ReadTimeoutError as e:
|
||
|
|
raise ReadTimeoutError(endpoint_url=request.url, error=e)
|
||
|
|
except ProtocolError as e:
|
||
|
|
raise ConnectionClosedError(
|
||
|
|
error=e,
|
||
|
|
request=request,
|
||
|
|
endpoint_url=request.url
|
||
|
|
)
|
||
|
|
except Exception as e:
|
||
|
|
message = 'Exception received when sending urllib3 HTTP request'
|
||
|
|
logger.debug(message, exc_info=True)
|
||
|
|
raise HTTPClientError(error=e)
|