1
# -*- coding: utf-8 -*-
7
This module contains the primary objects that power Requests.
13
from io import BytesIO, UnsupportedOperation
14
from .hooks import default_hooks
15
from .structures import CaseInsensitiveDict
17
from .auth import HTTPBasicAuth
18
from .cookies import cookiejar_from_dict, get_cookie_header
19
from .packages.urllib3.fields import RequestField
20
from .packages.urllib3.filepost import encode_multipart_formdata
21
from .packages.urllib3.util import parse_url
22
from .packages.urllib3.exceptions import DecodeError
23
from .exceptions import (
24
HTTPError, RequestException, MissingSchema, InvalidURL,
25
ChunkedEncodingError, ContentDecodingError)
27
guess_filename, get_auth_from_url, requote_uri,
28
stream_decode_response_unicode, to_key_val_list, parse_header_links,
29
iter_slices, guess_json_utf, super_len, to_native_string)
31
cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
32
is_py2, chardet, json, builtin_str, basestring, IncompleteRead)
33
from .status_codes import codes
35
#: The set of HTTP status codes that indicate an automatically
36
#: processable redirect.
41
codes.temporary_moved, # 307
43
DEFAULT_REDIRECT_LIMIT = 30
44
CONTENT_CHUNK_SIZE = 10 * 1024
48
class RequestEncodingMixin(object):
51
"""Build the path URL to use."""
55
p = urlsplit(self.url)
71
def _encode_params(data):
72
"""Encode parameters in a piece of data.
74
Will successfully encode parameters when passed as a dict or a list of
75
2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
76
if parameters are supplied as a dict.
79
if isinstance(data, (str, bytes)):
81
elif hasattr(data, 'read'):
83
elif hasattr(data, '__iter__'):
85
for k, vs in to_key_val_list(data):
86
if isinstance(vs, basestring) or not hasattr(vs, '__iter__'):
91
(k.encode('utf-8') if isinstance(k, str) else k,
92
v.encode('utf-8') if isinstance(v, str) else v))
93
return urlencode(result, doseq=True)
98
def _encode_files(files, data):
99
"""Build the body for a multipart/form-data request.
101
Will successfully encode files when passed as a dict or a list of
102
2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
103
if parameters are supplied as a dict.
107
raise ValueError("Files must be provided.")
108
elif isinstance(data, basestring):
109
raise ValueError("Data must not be a string.")
112
fields = to_key_val_list(data or {})
113
files = to_key_val_list(files or {})
115
for field, val in fields:
116
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
120
# Don't call str() on bytestrings: in Py3 it all goes wrong.
121
if not isinstance(v, bytes):
125
(field.decode('utf-8') if isinstance(field, bytes) else field,
126
v.encode('utf-8') if isinstance(v, str) else v))
129
# support for explicit filename
132
if isinstance(v, (tuple, list)):
140
fn = guess_filename(v) or k
142
if isinstance(fp, str):
144
if isinstance(fp, bytes):
147
rf = RequestField(name=k, data=fp.read(),
148
filename=fn, headers=fh)
149
rf.make_multipart(content_type=ft)
150
new_fields.append(rf)
152
body, content_type = encode_multipart_formdata(new_fields)
154
return body, content_type
157
class RequestHooksMixin(object):
158
def register_hook(self, event, hook):
159
"""Properly register a hook."""
161
if event not in self.hooks:
162
raise ValueError('Unsupported event specified, with event name "%s"' % (event))
164
if isinstance(hook, collections.Callable):
165
self.hooks[event].append(hook)
166
elif hasattr(hook, '__iter__'):
167
self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))
169
def deregister_hook(self, event, hook):
170
"""Deregister a previously registered hook.
171
Returns True if the hook existed, False if not.
175
self.hooks[event].remove(hook)
181
class Request(RequestHooksMixin):
182
"""A user-created :class:`Request <Request>` object.
184
Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
186
:param method: HTTP method to use.
187
:param url: URL to send.
188
:param headers: dictionary of headers to send.
189
:param files: dictionary of {filename: fileobject} files to multipart upload.
190
:param data: the body to attach the request. If a dictionary is provided, form-encoding will take place.
191
:param params: dictionary of URL parameters to append to the URL.
192
:param auth: Auth handler or (user, pass) tuple.
193
:param cookies: dictionary or CookieJar of cookies to attach to this request.
194
:param hooks: dictionary of callback hooks, for internal usage.
199
>>> req = requests.Request('GET', 'http://httpbin.org/get')
201
<PreparedRequest [GET]>
215
# Default empty dicts for dict params.
216
data = [] if data is None else data
217
files = [] if files is None else files
218
headers = {} if headers is None else headers
219
params = {} if params is None else params
220
hooks = {} if hooks is None else hooks
222
self.hooks = default_hooks()
223
for (k, v) in list(hooks.items()):
224
self.register_hook(event=k, hook=v)
228
self.headers = headers
233
self.cookies = cookies
236
return '<Request [%s]>' % (self.method)
239
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
240
p = PreparedRequest()
244
headers=self.headers,
249
cookies=self.cookies,
255
class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
256
"""The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
257
containing the exact bytes that will be sent to the server.
259
Generated from either a :class:`Request <Request>` object or manually.
264
>>> req = requests.Request('GET', 'http://httpbin.org/get')
265
>>> r = req.prepare()
266
<PreparedRequest [GET]>
268
>>> s = requests.Session()
275
#: HTTP verb to send to the server.
277
#: HTTP URL to send the request to.
279
#: dictionary of HTTP headers.
281
# The `CookieJar` used to create the Cookie header will be stored here
282
# after prepare_cookies is called
284
#: request body to send to the server.
286
#: dictionary of callback hooks, for internal usage.
287
self.hooks = default_hooks()
289
def prepare(self, method=None, url=None, headers=None, files=None,
290
data=None, params=None, auth=None, cookies=None, hooks=None):
291
"""Prepares the entire request with the given parameters."""
293
self.prepare_method(method)
294
self.prepare_url(url, params)
295
self.prepare_headers(headers)
296
self.prepare_cookies(cookies)
297
self.prepare_body(data, files)
298
self.prepare_auth(auth, url)
299
# Note that prepare_auth must be last to enable authentication schemes
300
# such as OAuth to work on a fully prepared request.
302
# This MUST go after prepare_auth. Authenticators could add a hook
303
self.prepare_hooks(hooks)
306
return '<PreparedRequest [%s]>' % (self.method)
309
p = PreparedRequest()
310
p.method = self.method
312
p.headers = self.headers.copy()
313
p._cookies = self._cookies.copy()
318
def prepare_method(self, method):
319
"""Prepares the given HTTP method."""
321
if self.method is not None:
322
self.method = self.method.upper()
324
def prepare_url(self, url, params):
325
"""Prepares the given HTTP URL."""
326
#: Accept objects that have string representations.
332
except UnicodeDecodeError:
335
# Don't do any URL preparation for oddball schemes
336
if ':' in url and not url.lower().startswith('http'):
340
# Support for unicode domain names and paths.
341
scheme, auth, host, port, path, query, fragment = parse_url(url)
344
raise MissingSchema("Invalid URL {0!r}: No schema supplied. "
345
"Perhaps you meant http://{0}?".format(url))
348
raise InvalidURL("Invalid URL %r: No host supplied" % url)
350
# Only want to apply IDNA to the hostname
352
host = host.encode('idna').decode('utf-8')
354
raise InvalidURL('URL has an invalid label.')
356
# Carefully reconstruct the network location
362
netloc += ':' + str(port)
364
# Bare domains aren't valid URLs.
369
if isinstance(scheme, str):
370
scheme = scheme.encode('utf-8')
371
if isinstance(netloc, str):
372
netloc = netloc.encode('utf-8')
373
if isinstance(path, str):
374
path = path.encode('utf-8')
375
if isinstance(query, str):
376
query = query.encode('utf-8')
377
if isinstance(fragment, str):
378
fragment = fragment.encode('utf-8')
380
enc_params = self._encode_params(params)
383
query = '%s&%s' % (query, enc_params)
387
url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
390
def prepare_headers(self, headers):
391
"""Prepares the given HTTP headers."""
394
self.headers = CaseInsensitiveDict((to_native_string(name), value) for name, value in headers.items())
396
self.headers = CaseInsensitiveDict()
398
def prepare_body(self, data, files):
399
"""Prepares the given HTTP body data."""
401
# Check if file, fo, generator, iterator.
402
# If not, run through normal process.
410
hasattr(data, '__iter__'),
411
not isinstance(data, (basestring, list, tuple, dict))
415
length = super_len(data)
416
except (TypeError, AttributeError, UnsupportedOperation):
423
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
425
if length is not None:
426
self.headers['Content-Length'] = builtin_str(length)
428
self.headers['Transfer-Encoding'] = 'chunked'
430
# Multi-part file uploads.
432
(body, content_type) = self._encode_files(files, data)
435
body = self._encode_params(data)
436
if isinstance(data, str) or isinstance(data, builtin_str) or hasattr(data, 'read'):
439
content_type = 'application/x-www-form-urlencoded'
441
self.prepare_content_length(body)
443
# Add content-type if it wasn't explicitly provided.
444
if (content_type) and (not 'content-type' in self.headers):
445
self.headers['Content-Type'] = content_type
449
def prepare_content_length(self, body):
450
if hasattr(body, 'seek') and hasattr(body, 'tell'):
452
self.headers['Content-Length'] = builtin_str(body.tell())
454
elif body is not None:
457
self.headers['Content-Length'] = builtin_str(l)
458
elif self.method not in ('GET', 'HEAD'):
459
self.headers['Content-Length'] = '0'
461
def prepare_auth(self, auth, url=''):
462
"""Prepares the given HTTP auth data."""
464
# If no Auth is explicitly provided, extract it from the URL first.
466
url_auth = get_auth_from_url(self.url)
467
auth = url_auth if any(url_auth) else None
470
if isinstance(auth, tuple) and len(auth) == 2:
471
# special-case basic HTTP auth
472
auth = HTTPBasicAuth(*auth)
474
# Allow auth to make its changes.
477
# Update self to reflect the auth changes.
478
self.__dict__.update(r.__dict__)
480
# Recompute Content-Length
481
self.prepare_content_length(self.body)
483
def prepare_cookies(self, cookies):
484
"""Prepares the given HTTP cookie data."""
486
if isinstance(cookies, cookielib.CookieJar):
487
self._cookies = cookies
489
self._cookies = cookiejar_from_dict(cookies)
491
cookie_header = get_cookie_header(self._cookies, self)
492
if cookie_header is not None:
493
self.headers['Cookie'] = cookie_header
495
def prepare_hooks(self, hooks):
496
"""Prepares the given hooks."""
498
self.register_hook(event, hooks[event])
501
class Response(object):
502
"""The :class:`Response <Response>` object, which contains a
503
server's response to an HTTP request.
520
super(Response, self).__init__()
522
self._content = False
523
self._content_consumed = False
525
#: Integer Code of responded HTTP Status, e.g. 404 or 200.
526
self.status_code = None
528
#: Case-insensitive Dictionary of Response Headers.
529
#: For example, ``headers['content-encoding']`` will return the
530
#: value of a ``'Content-Encoding'`` response header.
531
self.headers = CaseInsensitiveDict()
533
#: File-like object representation of response (for advanced usage).
534
#: Use of ``raw`` requires that ``stream=True`` be set on the request.
535
# This requirement does not apply for use internally to Requests.
538
#: Final URL location of Response.
541
#: Encoding to decode with when accessing r.text.
544
#: A list of :class:`Response <Response>` objects from
545
#: the history of the Request. Any redirect responses will end
546
#: up here. The list is sorted from the oldest to the most recent request.
549
#: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
552
#: A CookieJar of Cookies the server sent back.
553
self.cookies = cookiejar_from_dict({})
555
#: The amount of time elapsed between sending the request
556
#: and the arrival of the response (as a timedelta)
557
self.elapsed = datetime.timedelta(0)
559
def __getstate__(self):
560
# Consume everything; accessing the content attribute makes
561
# sure the content has been fully read.
562
if not self._content_consumed:
566
(attr, getattr(self, attr, None))
567
for attr in self.__attrs__
570
def __setstate__(self, state):
571
for name, value in state.items():
572
setattr(self, name, value)
574
# pickled objects do not have .raw
575
setattr(self, '_content_consumed', True)
576
setattr(self, 'raw', None)
579
return '<Response [%s]>' % (self.status_code)
582
"""Returns true if :attr:`status_code` is 'OK'."""
585
def __nonzero__(self):
586
"""Returns true if :attr:`status_code` is 'OK'."""
590
"""Allows you to use a response as an iterator."""
591
return self.iter_content(128)
596
self.raise_for_status()
597
except RequestException:
602
def is_redirect(self):
603
"""True if this Response is a well-formed HTTP redirect that could have
604
been processed automatically (by :meth:`Session.resolve_redirects`).
606
return ('location' in self.headers and self.status_code in REDIRECT_STATI)
609
def apparent_encoding(self):
610
"""The apparent encoding, provided by the chardet library"""
611
return chardet.detect(self.content)['encoding']
613
def iter_content(self, chunk_size=1, decode_unicode=False):
614
"""Iterates over the response data. When stream=True is set on the
615
request, this avoids reading the content at once into memory for
616
large responses. The chunk size is the number of bytes it should
617
read into memory. This is not necessarily the length of each item
618
returned as decoding can take place.
620
If decode_unicode is True, content will be decoded using the best
621
available encoding based on the response.
625
# Special case for urllib3.
627
for chunk in self.raw.stream(chunk_size, decode_content=True):
629
except IncompleteRead as e:
630
raise ChunkedEncodingError(e)
631
except DecodeError as e:
632
raise ContentDecodingError(e)
633
except AttributeError:
634
# Standard file-like object.
636
chunk = self.raw.read(chunk_size)
641
self._content_consumed = True
643
# simulate reading small chunks of the content
644
reused_chunks = iter_slices(self._content, chunk_size)
646
stream_chunks = generate()
648
chunks = reused_chunks if self._content_consumed else stream_chunks
651
chunks = stream_decode_response_unicode(chunks, self)
655
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None):
656
"""Iterates over the response data, one line at a time. When
657
stream=True is set on the request, this avoids reading the
658
content at once into memory for large responses.
663
for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode):
665
if pending is not None:
666
chunk = pending + chunk
667
lines = chunk.splitlines()
669
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
670
pending = lines.pop()
677
if pending is not None:
682
"""Content of the response, in bytes."""
684
if self._content is False:
687
if self._content_consumed:
689
'The content for this response was already consumed')
691
if self.status_code == 0:
694
self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
696
except AttributeError:
699
self._content_consumed = True
700
# don't need to release the connection; that's been handled by urllib3
701
# since we exhausted the data.
706
"""Content of the response, in unicode.
708
If Response.encoding is None, encoding will be guessed using
711
The encoding of the response content is determined based solely on HTTP
712
headers, following RFC 2616 to the letter. If you can take advantage of
713
non-HTTP knowledge to make a better guess at the encoding, you should
714
set ``r.encoding`` appropriately before accessing this property.
717
# Try charset from content-type
719
encoding = self.encoding
724
# Fallback to auto-detected encoding.
725
if self.encoding is None:
726
encoding = self.apparent_encoding
728
# Decode unicode from given encoding.
730
content = str(self.content, encoding, errors='replace')
731
except (LookupError, TypeError):
732
# A LookupError is raised if the encoding was not found which could
733
# indicate a misspelling or similar mistake.
735
# A TypeError can be raised if encoding is None
737
# So we try blindly encoding.
738
content = str(self.content, errors='replace')
742
def json(self, **kwargs):
743
"""Returns the json-encoded content of a response, if any.
745
:param \*\*kwargs: Optional arguments that ``json.loads`` takes.
748
if not self.encoding and len(self.content) > 3:
749
# No encoding set. JSON RFC 4627 section 3 states we should expect
750
# UTF-8, -16 or -32. Detect which one to use; If the detection or
751
# decoding fails, fall back to `self.text` (using chardet to make
753
encoding = guess_json_utf(self.content)
754
if encoding is not None:
756
return json.loads(self.content.decode(encoding), **kwargs)
757
except UnicodeDecodeError:
758
# Wrong UTF codec detected; usually because it's not UTF-8
759
# but some other 8-bit codec. This is an RFC violation,
760
# and the server didn't bother to tell us what codec *was*
763
return json.loads(self.text, **kwargs)
767
"""Returns the parsed header links of the response, if any."""
769
header = self.headers.get('link')
775
links = parse_header_links(header)
778
key = link.get('rel') or link.get('url')
783
def raise_for_status(self):
784
"""Raises stored :class:`HTTPError`, if one occurred."""
788
if 400 <= self.status_code < 500:
789
http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason)
791
elif 500 <= self.status_code < 600:
792
http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason)
795
raise HTTPError(http_error_msg, response=self)
798
"""Releases the connection back to the pool. Once this method has been
799
called the underlying ``raw`` object must not be accessed again.
801
*Note: Should not normally need to be called explicitly.*
803
return self.raw.release_conn()