1
# -*- coding: utf-8 -*-
7
This module contains the primary objects that power Requests.
13
from io import BytesIO, UnsupportedOperation
14
from .hooks import default_hooks
15
from .structures import CaseInsensitiveDict
17
from .auth import HTTPBasicAuth
18
from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
19
from .packages.urllib3.fields import RequestField
20
from .packages.urllib3.filepost import encode_multipart_formdata
21
from .packages.urllib3.util import parse_url
22
from .packages.urllib3.exceptions import (
23
DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
24
from .exceptions import (
25
HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
26
ContentDecodingError, ConnectionError, StreamConsumedError)
28
guess_filename, get_auth_from_url, requote_uri,
29
stream_decode_response_unicode, to_key_val_list, parse_header_links,
30
iter_slices, guess_json_utf, super_len, to_native_string)
32
cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
33
is_py2, chardet, builtin_str, basestring)
34
from .compat import json as complexjson
35
from .status_codes import codes
37
#: The set of HTTP status codes that indicate an automatically
38
#: processable redirect.
43
codes.temporary_redirect, # 307
44
codes.permanent_redirect, # 308
47
DEFAULT_REDIRECT_LIMIT = 30
48
CONTENT_CHUNK_SIZE = 10 * 1024
52
class RequestEncodingMixin(object):
55
"""Build the path URL to use."""
59
p = urlsplit(self.url)
75
def _encode_params(data):
76
"""Encode parameters in a piece of data.
78
Will successfully encode parameters when passed as a dict or a list of
79
2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
80
if parameters are supplied as a dict.
83
if isinstance(data, (str, bytes)):
85
elif hasattr(data, 'read'):
87
elif hasattr(data, '__iter__'):
89
for k, vs in to_key_val_list(data):
90
if isinstance(vs, basestring) or not hasattr(vs, '__iter__'):
95
(k.encode('utf-8') if isinstance(k, str) else k,
96
v.encode('utf-8') if isinstance(v, str) else v))
97
return urlencode(result, doseq=True)
102
def _encode_files(files, data):
103
"""Build the body for a multipart/form-data request.
105
Will successfully encode files when passed as a dict or a list of
106
tuples. Order is retained if data is a list of tuples but arbitrary
107
if parameters are supplied as a dict.
108
The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
109
or 4-tuples (filename, fileobj, contentype, custom_headers).
113
raise ValueError("Files must be provided.")
114
elif isinstance(data, basestring):
115
raise ValueError("Data must not be a string.")
118
fields = to_key_val_list(data or {})
119
files = to_key_val_list(files or {})
121
for field, val in fields:
122
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
126
# Don't call str() on bytestrings: in Py3 it all goes wrong.
127
if not isinstance(v, bytes):
131
(field.decode('utf-8') if isinstance(field, bytes) else field,
132
v.encode('utf-8') if isinstance(v, str) else v))
135
# support for explicit filename
138
if isinstance(v, (tuple, list)):
146
fn = guess_filename(v) or k
149
if isinstance(fp, (str, bytes, bytearray)):
154
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
155
rf.make_multipart(content_type=ft)
156
new_fields.append(rf)
158
body, content_type = encode_multipart_formdata(new_fields)
160
return body, content_type
163
class RequestHooksMixin(object):
164
def register_hook(self, event, hook):
165
"""Properly register a hook."""
167
if event not in self.hooks:
168
raise ValueError('Unsupported event specified, with event name "%s"' % (event))
170
if isinstance(hook, collections.Callable):
171
self.hooks[event].append(hook)
172
elif hasattr(hook, '__iter__'):
173
self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))
175
def deregister_hook(self, event, hook):
176
"""Deregister a previously registered hook.
177
Returns True if the hook existed, False if not.
181
self.hooks[event].remove(hook)
187
class Request(RequestHooksMixin):
188
"""A user-created :class:`Request <Request>` object.
190
Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
192
:param method: HTTP method to use.
193
:param url: URL to send.
194
:param headers: dictionary of headers to send.
195
:param files: dictionary of {filename: fileobject} files to multipart upload.
196
:param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place.
197
:param json: json for the body to attach to the request (if files or data is not specified).
198
:param params: dictionary of URL parameters to append to the URL.
199
:param auth: Auth handler or (user, pass) tuple.
200
:param cookies: dictionary or CookieJar of cookies to attach to this request.
201
:param hooks: dictionary of callback hooks, for internal usage.
206
>>> req = requests.Request('GET', 'http://httpbin.org/get')
208
<PreparedRequest [GET]>
211
def __init__(self, method=None, url=None, headers=None, files=None,
212
data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
214
# Default empty dicts for dict params.
215
data = [] if data is None else data
216
files = [] if files is None else files
217
headers = {} if headers is None else headers
218
params = {} if params is None else params
219
hooks = {} if hooks is None else hooks
221
self.hooks = default_hooks()
222
for (k, v) in list(hooks.items()):
223
self.register_hook(event=k, hook=v)
227
self.headers = headers
233
self.cookies = cookies
236
return '<Request [%s]>' % (self.method)
239
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
240
p = PreparedRequest()
244
headers=self.headers,
250
cookies=self.cookies,
256
class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
257
"""The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
258
containing the exact bytes that will be sent to the server.
260
Generated from either a :class:`Request <Request>` object or manually.
265
>>> req = requests.Request('GET', 'http://httpbin.org/get')
266
>>> r = req.prepare()
267
<PreparedRequest [GET]>
269
>>> s = requests.Session()
276
#: HTTP verb to send to the server.
278
#: HTTP URL to send the request to.
280
#: dictionary of HTTP headers.
282
# The `CookieJar` used to create the Cookie header will be stored here
283
# after prepare_cookies is called
285
#: request body to send to the server.
287
#: dictionary of callback hooks, for internal usage.
288
self.hooks = default_hooks()
290
def prepare(self, method=None, url=None, headers=None, files=None,
291
data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
292
"""Prepares the entire request with the given parameters."""
294
self.prepare_method(method)
295
self.prepare_url(url, params)
296
self.prepare_headers(headers)
297
self.prepare_cookies(cookies)
298
self.prepare_body(data, files, json)
299
self.prepare_auth(auth, url)
301
# Note that prepare_auth must be last to enable authentication schemes
302
# such as OAuth to work on a fully prepared request.
304
# This MUST go after prepare_auth. Authenticators could add a hook
305
self.prepare_hooks(hooks)
308
return '<PreparedRequest [%s]>' % (self.method)
311
p = PreparedRequest()
312
p.method = self.method
314
p.headers = self.headers.copy() if self.headers is not None else None
315
p._cookies = _copy_cookie_jar(self._cookies)
320
def prepare_method(self, method):
321
"""Prepares the given HTTP method."""
323
if self.method is not None:
324
self.method = to_native_string(self.method.upper())
326
def prepare_url(self, url, params):
327
"""Prepares the given HTTP URL."""
328
#: Accept objects that have string representations.
329
#: We're unable to blindly call unicode/str functions
330
#: as this will include the bytestring indicator (b'')
332
#: https://github.com/kennethreitz/requests/pull/2238
333
if isinstance(url, bytes):
334
url = url.decode('utf8')
336
url = unicode(url) if is_py2 else str(url)
338
# Don't do any URL preparation for non-HTTP schemes like `mailto`,
339
# `data` etc to work around exceptions from `url_parse`, which
340
# handles RFC 3986 only.
341
if ':' in url and not url.lower().startswith('http'):
345
# Support for unicode domain names and paths.
347
scheme, auth, host, port, path, query, fragment = parse_url(url)
348
except LocationParseError as e:
349
raise InvalidURL(*e.args)
352
error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?")
353
error = error.format(to_native_string(url, 'utf8'))
355
raise MissingSchema(error)
358
raise InvalidURL("Invalid URL %r: No host supplied" % url)
360
# Only want to apply IDNA to the hostname
362
host = host.encode('idna').decode('utf-8')
364
raise InvalidURL('URL has an invalid label.')
366
# Carefully reconstruct the network location
372
netloc += ':' + str(port)
374
# Bare domains aren't valid URLs.
379
if isinstance(scheme, str):
380
scheme = scheme.encode('utf-8')
381
if isinstance(netloc, str):
382
netloc = netloc.encode('utf-8')
383
if isinstance(path, str):
384
path = path.encode('utf-8')
385
if isinstance(query, str):
386
query = query.encode('utf-8')
387
if isinstance(fragment, str):
388
fragment = fragment.encode('utf-8')
390
if isinstance(params, (str, bytes)):
391
params = to_native_string(params)
393
enc_params = self._encode_params(params)
396
query = '%s&%s' % (query, enc_params)
400
url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
403
def prepare_headers(self, headers):
404
"""Prepares the given HTTP headers."""
407
self.headers = CaseInsensitiveDict((to_native_string(name), value) for name, value in headers.items())
409
self.headers = CaseInsensitiveDict()
411
def prepare_body(self, data, files, json=None):
412
"""Prepares the given HTTP body data."""
414
# Check if file, fo, generator, iterator.
415
# If not, run through normal process.
422
if not data and json is not None:
423
content_type = 'application/json'
424
body = complexjson.dumps(json)
427
hasattr(data, '__iter__'),
428
not isinstance(data, (basestring, list, tuple, dict))
432
length = super_len(data)
433
except (TypeError, AttributeError, UnsupportedOperation):
440
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
443
self.headers['Content-Length'] = builtin_str(length)
445
self.headers['Transfer-Encoding'] = 'chunked'
447
# Multi-part file uploads.
449
(body, content_type) = self._encode_files(files, data)
452
body = self._encode_params(data)
453
if isinstance(data, basestring) or hasattr(data, 'read'):
456
content_type = 'application/x-www-form-urlencoded'
458
self.prepare_content_length(body)
460
# Add content-type if it wasn't explicitly provided.
461
if content_type and ('content-type' not in self.headers):
462
self.headers['Content-Type'] = content_type
466
def prepare_content_length(self, body):
467
if hasattr(body, 'seek') and hasattr(body, 'tell'):
468
curr_pos = body.tell()
470
end_pos = body.tell()
471
self.headers['Content-Length'] = builtin_str(max(0, end_pos - curr_pos))
472
body.seek(curr_pos, 0)
473
elif body is not None:
476
self.headers['Content-Length'] = builtin_str(l)
477
elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None):
478
self.headers['Content-Length'] = '0'
480
def prepare_auth(self, auth, url=''):
481
"""Prepares the given HTTP auth data."""
483
# If no Auth is explicitly provided, extract it from the URL first.
485
url_auth = get_auth_from_url(self.url)
486
auth = url_auth if any(url_auth) else None
489
if isinstance(auth, tuple) and len(auth) == 2:
490
# special-case basic HTTP auth
491
auth = HTTPBasicAuth(*auth)
493
# Allow auth to make its changes.
496
# Update self to reflect the auth changes.
497
self.__dict__.update(r.__dict__)
499
# Recompute Content-Length
500
self.prepare_content_length(self.body)
502
def prepare_cookies(self, cookies):
503
"""Prepares the given HTTP cookie data.
505
This function eventually generates a ``Cookie`` header from the
506
given cookies using cookielib. Due to cookielib's design, the header
507
will not be regenerated if it already exists, meaning this function
508
can only be called once for the life of the
509
:class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
510
to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
511
header is removed beforehand."""
513
if isinstance(cookies, cookielib.CookieJar):
514
self._cookies = cookies
516
self._cookies = cookiejar_from_dict(cookies)
518
cookie_header = get_cookie_header(self._cookies, self)
519
if cookie_header is not None:
520
self.headers['Cookie'] = cookie_header
522
def prepare_hooks(self, hooks):
523
"""Prepares the given hooks."""
524
# hooks can be passed as None to the prepare method and to this
525
# method. To prevent iterating over None, simply use an empty list
526
# if hooks is False-y
529
self.register_hook(event, hooks[event])
532
class Response(object):
533
"""The :class:`Response <Response>` object, which contains a
534
server's response to an HTTP request.
538
'_content', 'status_code', 'headers', 'url', 'history',
539
'encoding', 'reason', 'cookies', 'elapsed', 'request'
543
super(Response, self).__init__()
545
self._content = False
546
self._content_consumed = False
548
#: Integer Code of responded HTTP Status, e.g. 404 or 200.
549
self.status_code = None
551
#: Case-insensitive Dictionary of Response Headers.
552
#: For example, ``headers['content-encoding']`` will return the
553
#: value of a ``'Content-Encoding'`` response header.
554
self.headers = CaseInsensitiveDict()
556
#: File-like object representation of response (for advanced usage).
557
#: Use of ``raw`` requires that ``stream=True`` be set on the request.
558
# This requirement does not apply for use internally to Requests.
561
#: Final URL location of Response.
564
#: Encoding to decode with when accessing r.text.
567
#: A list of :class:`Response <Response>` objects from
568
#: the history of the Request. Any redirect responses will end
569
#: up here. The list is sorted from the oldest to the most recent request.
572
#: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
575
#: A CookieJar of Cookies the server sent back.
576
self.cookies = cookiejar_from_dict({})
578
#: The amount of time elapsed between sending the request
579
#: and the arrival of the response (as a timedelta).
580
#: This property specifically measures the time taken between sending
581
#: the first byte of the request and finishing parsing the headers. It
582
#: is therefore unaffected by consuming the response content or the
583
#: value of the ``stream`` keyword argument.
584
self.elapsed = datetime.timedelta(0)
586
#: The :class:`PreparedRequest <PreparedRequest>` object to which this
590
def __getstate__(self):
591
# Consume everything; accessing the content attribute makes
592
# sure the content has been fully read.
593
if not self._content_consumed:
597
(attr, getattr(self, attr, None))
598
for attr in self.__attrs__
601
def __setstate__(self, state):
602
for name, value in state.items():
603
setattr(self, name, value)
605
# pickled objects do not have .raw
606
setattr(self, '_content_consumed', True)
607
setattr(self, 'raw', None)
610
return '<Response [%s]>' % (self.status_code)
613
"""Returns true if :attr:`status_code` is 'OK'."""
616
def __nonzero__(self):
617
"""Returns true if :attr:`status_code` is 'OK'."""
621
"""Allows you to use a response as an iterator."""
622
return self.iter_content(128)
627
self.raise_for_status()
633
def is_redirect(self):
634
"""True if this Response is a well-formed HTTP redirect that could have
635
been processed automatically (by :meth:`Session.resolve_redirects`).
637
return ('location' in self.headers and self.status_code in REDIRECT_STATI)
640
def is_permanent_redirect(self):
641
"""True if this Response one of the permanent versions of redirect"""
642
return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))
645
def apparent_encoding(self):
646
"""The apparent encoding, provided by the chardet library"""
647
return chardet.detect(self.content)['encoding']
649
def iter_content(self, chunk_size=1, decode_unicode=False):
650
"""Iterates over the response data. When stream=True is set on the
651
request, this avoids reading the content at once into memory for
652
large responses. The chunk size is the number of bytes it should
653
read into memory. This is not necessarily the length of each item
654
returned as decoding can take place.
656
If decode_unicode is True, content will be decoded using the best
657
available encoding based on the response.
661
# Special case for urllib3.
662
if hasattr(self.raw, 'stream'):
664
for chunk in self.raw.stream(chunk_size, decode_content=True):
666
except ProtocolError as e:
667
raise ChunkedEncodingError(e)
668
except DecodeError as e:
669
raise ContentDecodingError(e)
670
except ReadTimeoutError as e:
671
raise ConnectionError(e)
673
# Standard file-like object.
675
chunk = self.raw.read(chunk_size)
680
self._content_consumed = True
682
if self._content_consumed and isinstance(self._content, bool):
683
raise StreamConsumedError()
684
# simulate reading small chunks of the content
685
reused_chunks = iter_slices(self._content, chunk_size)
687
stream_chunks = generate()
689
chunks = reused_chunks if self._content_consumed else stream_chunks
692
chunks = stream_decode_response_unicode(chunks, self)
696
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
697
"""Iterates over the response data, one line at a time. When
698
stream=True is set on the request, this avoids reading the
699
content at once into memory for large responses.
701
.. note:: This method is not reentrant safe.
706
for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode):
708
if pending is not None:
709
chunk = pending + chunk
712
lines = chunk.split(delimiter)
714
lines = chunk.splitlines()
716
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
717
pending = lines.pop()
724
if pending is not None:
729
"""Content of the response, in bytes."""
731
if self._content is False:
734
if self._content_consumed:
736
'The content for this response was already consumed')
738
if self.status_code == 0:
741
self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
743
except AttributeError:
746
self._content_consumed = True
747
# don't need to release the connection; that's been handled by urllib3
748
# since we exhausted the data.
753
"""Content of the response, in unicode.
755
If Response.encoding is None, encoding will be guessed using
758
The encoding of the response content is determined based solely on HTTP
759
headers, following RFC 2616 to the letter. If you can take advantage of
760
non-HTTP knowledge to make a better guess at the encoding, you should
761
set ``r.encoding`` appropriately before accessing this property.
764
# Try charset from content-type
766
encoding = self.encoding
771
# Fallback to auto-detected encoding.
772
if self.encoding is None:
773
encoding = self.apparent_encoding
775
# Decode unicode from given encoding.
777
content = str(self.content, encoding, errors='replace')
778
except (LookupError, TypeError):
779
# A LookupError is raised if the encoding was not found which could
780
# indicate a misspelling or similar mistake.
782
# A TypeError can be raised if encoding is None
784
# So we try blindly encoding.
785
content = str(self.content, errors='replace')
789
def json(self, **kwargs):
790
"""Returns the json-encoded content of a response, if any.
792
:param \*\*kwargs: Optional arguments that ``json.loads`` takes.
795
if not self.encoding and len(self.content) > 3:
796
# No encoding set. JSON RFC 4627 section 3 states we should expect
797
# UTF-8, -16 or -32. Detect which one to use; If the detection or
798
# decoding fails, fall back to `self.text` (using chardet to make
800
encoding = guess_json_utf(self.content)
801
if encoding is not None:
803
return complexjson.loads(
804
self.content.decode(encoding), **kwargs
806
except UnicodeDecodeError:
807
# Wrong UTF codec detected; usually because it's not UTF-8
808
# but some other 8-bit codec. This is an RFC violation,
809
# and the server didn't bother to tell us what codec *was*
812
return complexjson.loads(self.text, **kwargs)
816
"""Returns the parsed header links of the response, if any."""
818
header = self.headers.get('link')
824
links = parse_header_links(header)
827
key = link.get('rel') or link.get('url')
832
def raise_for_status(self):
833
"""Raises stored :class:`HTTPError`, if one occurred."""
837
if 400 <= self.status_code < 500:
838
http_error_msg = '%s Client Error: %s for url: %s' % (self.status_code, self.reason, self.url)
840
elif 500 <= self.status_code < 600:
841
http_error_msg = '%s Server Error: %s for url: %s' % (self.status_code, self.reason, self.url)
844
raise HTTPError(http_error_msg, response=self)
847
"""Releases the connection back to the pool. Once this method has been
848
called the underlying ``raw`` object must not be accessed again.
850
*Note: Should not normally need to be called explicitly.*
852
if not self._content_consumed:
853
return self.raw.close()
855
return self.raw.release_conn()
added
removed
.tox/py35/lib/python3.5/site-packages/pip/_vendor/requests/models.py
