1
# urllib3/connectionpool.py
2
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
4
# This module is part of urllib3 and is released under
5
# the MIT License: http://www.opensource.org/licenses/mit-license.php
10
from socket import error as SocketError, timeout as SocketTimeout
14
from queue import LifoQueue, Empty, Full
16
from Queue import LifoQueue, Empty, Full
17
import Queue as _ # Platform-specific: Windows
20
from .exceptions import (
32
from .packages.ssl_match_hostname import CertificateError
33
from .packages import six
34
from .connection import (
37
HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
38
HTTPException, BaseSSLError,
40
from .request import RequestMethods
41
from .response import HTTPResponse
45
is_connection_dropped,
50
xrange = six.moves.xrange
52
log = logging.getLogger(__name__)
58
class ConnectionPool(object):
60
Base class for all connection pools, such as
61
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
67
def __init__(self, host, port=None):
68
# httplib doesn't like it when we include brackets in ipv6 addresses
69
host = host.strip('[]')
75
return '%s(host=%r, port=%r)' % (type(self).__name__,
78
# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
79
_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK])
81
class HTTPConnectionPool(ConnectionPool, RequestMethods):
83
Thread-safe connection pool for one host.
86
Host used for this HTTP Connection (e.g. "localhost"), passed into
87
:class:`httplib.HTTPConnection`.
90
Port used for this HTTP Connection (None is equivalent to 80), passed
91
into :class:`httplib.HTTPConnection`.
94
Causes BadStatusLine to be raised if the status line can't be parsed
95
as a valid HTTP/1.0 or 1.1 status line, passed into
96
:class:`httplib.HTTPConnection`.
99
Only works in Python 2. This parameter is ignored in Python 3.
102
Socket timeout in seconds for each individual connection. This can
103
be a float or integer, which sets the timeout for the HTTP request,
104
or an instance of :class:`urllib3.util.Timeout` which gives you more
105
fine-grained control over request timeouts. After the constructor has
106
been parsed, this is always a `urllib3.util.Timeout` object.
109
Number of connections to save that can be reused. More than 1 is useful
110
in multithreaded situations. If ``block`` is set to false, more
111
connections will be created but they will not be saved once they've
115
If set to True, no more than ``maxsize`` connections will be used at
116
a time. When no free connections are available, the call will block
117
until a connection has been released. This is a useful side effect for
118
particular multithreaded situations where one does not want to use more
119
than maxsize connections per host to prevent flooding.
122
Headers to include with all requests, unless other headers are given
126
Parsed proxy URL, should not be used directly, instead, see
127
:class:`urllib3.connectionpool.ProxyManager`"
129
:param _proxy_headers:
130
A dictionary with proxy headers, should not be used directly,
131
instead, see :class:`urllib3.connectionpool.ProxyManager`"
135
ConnectionCls = HTTPConnection
137
def __init__(self, host, port=None, strict=False,
138
timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False,
139
headers=None, _proxy=None, _proxy_headers=None):
140
ConnectionPool.__init__(self, host, port)
141
RequestMethods.__init__(self, headers)
145
# This is for backwards compatibility and can be removed once a timeout
146
# can only be set to a Timeout object
147
if not isinstance(timeout, Timeout):
148
timeout = Timeout.from_float(timeout)
150
self.timeout = timeout
152
self.pool = self.QueueCls(maxsize)
156
self.proxy_headers = _proxy_headers or {}
158
# Fill the queue up so that doing get() on it will block properly
159
for _ in xrange(maxsize):
162
# These are mostly for testing and debugging purposes.
163
self.num_connections = 0
164
self.num_requests = 0
168
Return a fresh :class:`HTTPConnection`.
170
self.num_connections += 1
171
log.info("Starting new HTTP connection (%d): %s" %
172
(self.num_connections, self.host))
174
conn = self.ConnectionCls(host=self.host, port=self.port,
175
timeout=self.timeout.connect_timeout,
177
if self.proxy is not None:
178
# Enable Nagle's algorithm for proxies, to avoid packet
183
def _get_conn(self, timeout=None):
185
Get a connection. Will return a pooled connection if one is available.
187
If no connections are available and :prop:`.block` is ``False``, then a
188
fresh connection is returned.
191
Seconds to wait before giving up and raising
192
:class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
193
:prop:`.block` is ``True``.
197
conn = self.pool.get(block=self.block, timeout=timeout)
199
except AttributeError: # self.pool is None
200
raise ClosedPoolError(self, "Pool is closed.")
204
raise EmptyPoolError(self,
205
"Pool reached maximum size and no more "
206
"connections are allowed.")
207
pass # Oh well, we'll create a new connection then
209
# If this is a persistent connection, check if it got disconnected
210
if conn and is_connection_dropped(conn):
211
log.info("Resetting dropped connection: %s" % self.host)
214
return conn or self._new_conn()
216
def _put_conn(self, conn):
218
Put a connection back into the pool.
221
Connection object for the current host and port as returned by
222
:meth:`._new_conn` or :meth:`._get_conn`.
224
If the pool is already full, the connection is closed and discarded
225
because we exceeded maxsize. If connections are discarded frequently,
226
then maxsize should be increased.
228
If the pool is closed, then the connection will be closed and discarded.
231
self.pool.put(conn, block=False)
232
return # Everything is dandy, done.
233
except AttributeError:
237
# This should never happen if self.block == True
239
"Connection pool is full, discarding connection: %s" %
242
# Connection never got put back into the pool, close it.
246
def _get_timeout(self, timeout):
247
""" Helper that always returns a :class:`urllib3.util.Timeout` """
248
if timeout is _Default:
249
return self.timeout.clone()
251
if isinstance(timeout, Timeout):
252
return timeout.clone()
254
# User passed us an int/float. This is for backwards compatibility,
255
# can be removed later
256
return Timeout.from_float(timeout)
258
def _make_request(self, conn, method, url, timeout=_Default,
259
**httplib_request_kw):
261
Perform a request on a given urllib connection object taken from our
265
a connection from one of our connection pools
268
Socket timeout in seconds for the request. This can be a
269
float or integer, which will set the same timeout value for
270
the socket connect and the socket read, or an instance of
271
:class:`urllib3.util.Timeout`, which gives you more fine-grained
272
control over your timeouts.
274
self.num_requests += 1
276
timeout_obj = self._get_timeout(timeout)
279
timeout_obj.start_connect()
280
conn.timeout = timeout_obj.connect_timeout
281
# conn.request() calls httplib.*.request, not the method in
282
# urllib3.request. It also calls makefile (recv) on the socket.
283
conn.request(method, url, **httplib_request_kw)
284
except SocketTimeout:
285
raise ConnectTimeoutError(
286
self, "Connection to %s timed out. (connect timeout=%s)" %
287
(self.host, timeout_obj.connect_timeout))
289
# Reset the timeout for the recv() on the socket
290
read_timeout = timeout_obj.read_timeout
292
# App Engine doesn't have a sock attr
293
if hasattr(conn, 'sock'):
294
# In Python 3 socket.py will catch EAGAIN and return None when you
295
# try and read into the file pointer created by http.client, which
296
# instead raises a BadStatusLine exception. Instead of catching
297
# the exception and assuming all BadStatusLine exceptions are read
298
# timeouts, check for a zero timeout before making the request.
299
if read_timeout == 0:
300
raise ReadTimeoutError(
302
"Read timed out. (read timeout=%s)" % read_timeout)
303
if read_timeout is Timeout.DEFAULT_TIMEOUT:
304
conn.sock.settimeout(socket.getdefaulttimeout())
305
else: # None or a value
306
conn.sock.settimeout(read_timeout)
308
# Receive the response from the server
310
try: # Python 2.7+, use buffering of HTTP responses
311
httplib_response = conn.getresponse(buffering=True)
312
except TypeError: # Python 2.6 and older
313
httplib_response = conn.getresponse()
314
except SocketTimeout:
315
raise ReadTimeoutError(
316
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
318
except BaseSSLError as e:
319
# Catch possible read timeouts thrown as SSL errors. If not the
320
# case, rethrow the original. We need to do this because of:
321
# http://bugs.python.org/issue10272
322
if 'timed out' in str(e) or \
323
'did not complete (read)' in str(e): # Python 2.6
324
raise ReadTimeoutError(self, url, "Read timed out.")
328
except SocketError as e: # Platform-specific: Python 2
329
# See the above comment about EAGAIN in Python 3. In Python 2 we
330
# have to specifically catch it and throw the timeout error
331
if e.errno in _blocking_errnos:
332
raise ReadTimeoutError(
334
"Read timed out. (read timeout=%s)" % read_timeout)
338
# AppEngine doesn't have a version attr.
339
http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
340
log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
341
httplib_response.status,
342
httplib_response.length))
343
return httplib_response
347
Close all pooled connections and disable the pool.
349
# Disable access to the pool
350
old_pool, self.pool = self.pool, None
354
conn = old_pool.get(block=False)
361
def is_same_host(self, url):
363
Check if the given ``url`` is a member of the same host as this
366
if url.startswith('/'):
369
# TODO: Add optional support for socket.gethostbyname checking.
370
scheme, host, port = get_host(url)
372
# Use explicit default port for comparison when none is given
373
if self.port and not port:
374
port = port_by_scheme.get(scheme)
375
elif not self.port and port == port_by_scheme.get(scheme):
378
return (scheme, host, port) == (self.scheme, self.host, self.port)
380
def urlopen(self, method, url, body=None, headers=None, retries=3,
381
redirect=True, assert_same_host=True, timeout=_Default,
382
pool_timeout=None, release_conn=None, **response_kw):
384
Get a connection from the pool and perform an HTTP request. This is the
385
lowest level call for making a request, so you'll need to specify all
390
More commonly, it's appropriate to use a convenience method provided
391
by :class:`.RequestMethods`, such as :meth:`request`.
395
`release_conn` will only behave as expected if
396
`preload_content=False` because we want to make
397
`preload_content=False` the default behaviour someday soon without
398
breaking backwards compatibility.
401
HTTP request method (such as GET, POST, PUT, etc.)
404
Data to send in the request body (useful for creating
405
POST requests, see HTTPConnectionPool.post_url for
409
Dictionary of custom headers to send, such as User-Agent,
410
If-None-Match, etc. If None, pool headers are used. If provided,
411
these headers completely replace any pool-specific headers.
414
Number of retries to allow before raising a MaxRetryError exception.
415
If `False`, then retries are disabled and any exception is raised
419
If True, automatically handle redirects (status codes 301, 302,
420
303, 307, 308). Each redirect counts as a retry. Disabling retries
421
will disable redirect, too.
423
:param assert_same_host:
424
If ``True``, will make sure that the host of the pool requests is
425
consistent else will raise HostChangedError. When False, you can
426
use the pool on an HTTP proxy and request foreign hosts.
429
If specified, overrides the default timeout for this one
430
request. It may be a float (in seconds) or an instance of
431
:class:`urllib3.util.Timeout`.
434
If set and the pool is set to block=True, then this method will
435
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
436
connection is available within the time period.
439
If False, then the urlopen call will not release the connection
440
back into the pool once a response is received (but will release if
441
you read the entire contents of the response such as when
442
`preload_content=True`). This is useful if you're not preloading
443
the response's content immediately. You will need to call
444
``r.release_conn()`` on the response ``r`` to return the connection
445
back into the pool. If None, it takes the value of
446
``response_kw.get('preload_content', True)``.
448
:param \**response_kw:
449
Additional parameters are passed to
450
:meth:`urllib3.response.HTTPResponse.from_httplib`
453
headers = self.headers
455
if retries < 0 and retries is not False:
456
raise MaxRetryError(self, url)
458
if release_conn is None:
459
release_conn = response_kw.get('preload_content', True)
462
if assert_same_host and not self.is_same_host(url):
463
raise HostChangedError(self, url, retries - 1)
467
# Merge the proxy headers. Only do this in HTTP. We have to copy the
468
# headers dict so we can safely change it without those changes being
469
# reflected in anyone else's copy.
470
if self.scheme == 'http':
471
headers = headers.copy()
472
headers.update(self.proxy_headers)
474
# Must keep the exception bound to a separate variable or else Python 3
475
# complains about UnboundLocalError.
479
# Request a connection from the queue
480
conn = self._get_conn(timeout=pool_timeout)
482
# Make the request on the httplib connection object
483
httplib_response = self._make_request(conn, method, url,
485
body=body, headers=headers)
487
# If we're going to release the connection in ``finally:``, then
488
# the request doesn't need to know about the connection. Otherwise
489
# it will also try to release it and we'll have a double-release
491
response_conn = not release_conn and conn
493
# Import httplib's response into our own wrapper object
494
response = HTTPResponse.from_httplib(httplib_response,
496
connection=response_conn,
500
# The connection will be put back into the pool when
501
# ``response.release_conn()`` is called (implicitly by
502
# ``response.read()``)
505
# Timed out by queue.
506
raise EmptyPoolError(self, "No pool connections are available.")
508
except (BaseSSLError, CertificateError) as e:
509
# Release connection unconditionally because there is no way to
510
# close it externally in case of exception.
514
except (TimeoutError, HTTPException, SocketError) as e:
516
# Discard the connection for these exceptions. It will be
517
# be replaced during the next _get_conn() call.
522
if isinstance(e, TimeoutError):
523
# TimeoutError is exempt from MaxRetryError-wrapping.
524
# FIXME: ... Not sure why. Add a reason here.
527
# Wrap unexpected exceptions with the most appropriate
528
# module-level exception and re-raise.
529
if isinstance(e, SocketError) and self.proxy:
530
raise ProxyError('Cannot connect to proxy.', e)
533
raise ConnectionError('Connection failed.', e)
535
raise MaxRetryError(self, url, e)
537
# Keep track of the error for the retry warning.
542
# Put the connection back to be reused. If the connection is
543
# expired then it will be None, which will get replaced with a
544
# fresh connection during _get_conn.
549
log.warning("Retrying (%d attempts remain) after connection "
550
"broken by '%r': %s" % (retries, err, url))
551
return self.urlopen(method, url, body, headers, retries - 1,
552
redirect, assert_same_host,
553
timeout=timeout, pool_timeout=pool_timeout,
554
release_conn=release_conn, **response_kw)
557
redirect_location = redirect and response.get_redirect_location()
558
if redirect_location and retries is not False:
559
if response.status == 303:
561
log.info("Redirecting %s -> %s" % (url, redirect_location))
562
return self.urlopen(method, redirect_location, body, headers,
563
retries - 1, redirect, assert_same_host,
564
timeout=timeout, pool_timeout=pool_timeout,
565
release_conn=release_conn, **response_kw)
570
class HTTPSConnectionPool(HTTPConnectionPool):
572
Same as :class:`.HTTPConnectionPool`, but HTTPS.
574
When Python is compiled with the :mod:`ssl` module, then
575
:class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
576
instead of :class:`.HTTPSConnection`.
578
:class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
579
``assert_hostname`` and ``host`` in this order to verify connections.
580
If ``assert_hostname`` is False, no verification is done.
582
The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and
583
``ssl_version`` are only used if :mod:`ssl` is available and are fed into
584
:meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket
589
ConnectionCls = HTTPSConnection
591
def __init__(self, host, port=None,
592
strict=False, timeout=None, maxsize=1,
593
block=False, headers=None,
594
_proxy=None, _proxy_headers=None,
595
key_file=None, cert_file=None, cert_reqs=None,
596
ca_certs=None, ssl_version=None,
597
assert_hostname=None, assert_fingerprint=None):
599
HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize,
600
block, headers, _proxy, _proxy_headers)
601
self.key_file = key_file
602
self.cert_file = cert_file
603
self.cert_reqs = cert_reqs
604
self.ca_certs = ca_certs
605
self.ssl_version = ssl_version
606
self.assert_hostname = assert_hostname
607
self.assert_fingerprint = assert_fingerprint
609
def _prepare_conn(self, conn):
611
Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
612
and establish the tunnel if proxy is used.
615
if isinstance(conn, VerifiedHTTPSConnection):
616
conn.set_cert(key_file=self.key_file,
617
cert_file=self.cert_file,
618
cert_reqs=self.cert_reqs,
619
ca_certs=self.ca_certs,
620
assert_hostname=self.assert_hostname,
621
assert_fingerprint=self.assert_fingerprint)
622
conn.ssl_version = self.ssl_version
624
if self.proxy is not None:
627
set_tunnel = conn.set_tunnel
628
except AttributeError: # Platform-specific: Python 2.6
629
set_tunnel = conn._set_tunnel
630
set_tunnel(self.host, self.port, self.proxy_headers)
631
# Establish tunnel connection early, because otherwise httplib
632
# would improperly set Host: header to proxy's IP:port.
639
Return a fresh :class:`httplib.HTTPSConnection`.
641
self.num_connections += 1
642
log.info("Starting new HTTPS connection (%d): %s"
643
% (self.num_connections, self.host))
645
if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
646
# Platform-specific: Python without ssl
647
raise SSLError("Can't connect to HTTPS URL because the SSL "
648
"module is not available.")
650
actual_host = self.host
651
actual_port = self.port
652
if self.proxy is not None:
653
actual_host = self.proxy.host
654
actual_port = self.proxy.port
657
if not six.PY3: # Python 2
658
extra_params['strict'] = self.strict
660
conn = self.ConnectionCls(host=actual_host, port=actual_port,
661
timeout=self.timeout.connect_timeout,
663
if self.proxy is not None:
664
# Enable Nagle's algorithm for proxies, to avoid packet
668
return self._prepare_conn(conn)
671
def connection_from_url(url, **kw):
673
Given a url, return an :class:`.ConnectionPool` instance of its host.
675
This is a shortcut for not having to parse out the scheme, host, and port
676
of the url before creating an :class:`.ConnectionPool` instance.
679
Absolute URL string that must include the scheme. Port is optional.
682
Passes additional parameters to the constructor of the appropriate
683
:class:`.ConnectionPool`. Useful for specifying things like
684
timeout, maxsize, headers, etc.
688
>>> conn = connection_from_url('http://google.com/')
689
>>> r = conn.request('GET', '/')
691
scheme, host, port = get_host(url)
692
if scheme == 'https':
693
return HTTPSConnectionPool(host, port=port, **kw)
695
return HTTPConnectionPool(host, port=port, **kw)
added
removed
urllib3/connectionpool.py
