~ubuntu-branches/ubuntu/trusty/requests/trusty-proposed : revision 11

7

This module contains the primary objects that power Requests.

8

"""

9

10

import json

10

11

import os

11

12

from datetime import datetime

12

13

15

16

from .status_codes import codes

16

17

18

from .auth import HTTPBasicAuth, HTTPProxyAuth

19

from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header

18

20

from .packages.urllib3.response import HTTPResponse

19

21

from .packages.urllib3.exceptions import MaxRetryError, LocationParseError

20

22

from .packages.urllib3.exceptions import SSLError as _SSLError

27

29

URLRequired, SSLError, MissingSchema, InvalidSchema, InvalidURL)

28

30

from .utils import (

29

31

get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,

30

dict_from_string, stream_decode_response_unicode, get_netrc_auth,

32

stream_decode_response_unicode, get_netrc_auth, get_environ_proxies,

31

33

DEFAULT_CA_BUNDLE_PATH)

32

34

from .compat import (

33

urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,

34

SimpleCookie, is_py2)

35

cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,

36

StringIO, is_py2)

35

37

36

38

# Import chardet if it is available.

37

39

try:

38

40

import chardet

41

# hush pyflakes

42

chardet

39

43

except ImportError:

40

pass

44

chardet = None

41

45

42

46

REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)

43

47

CONTENT_CHUNK_SIZE = 10 * 1024

44

48

45

49

class Request(object):

46

50

"""The :class:`Request <Request>` object. It carries out all functionality of

109

113

# If no proxies are given, allow configuration by environment variables

110

114

# HTTP_PROXY and HTTPS_PROXY.

111

115

if not self.proxies and self.config.get('trust_env'):

112

if 'HTTP_PROXY' in os.environ:

113

self.proxies['http'] = os.environ['HTTP_PROXY']

114

if 'HTTPS_PROXY' in os.environ:

115

self.proxies['https'] = os.environ['HTTPS_PROXY']

116

self.proxies = get_environ_proxies()

116

117

self.data, self._enc_data = self._encode_params(data)

118

self.params, self._enc_params = self._encode_params(params)

119

self.files, self._enc_files = self._encode_files(files)

118

self.data = data

119

self.params = params

120

self.files = files

120

121

122

#: :class:`Response <Response>` instance, containing

122

123

#: content and metadata of HTTP Response, once :attr:`sent <send>`.

126

127

self.auth = auth

127

128

129

#: CookieJar to attach to :class:`Request <Request>`.

129

self.cookies = dict(cookies or [])

130

if isinstance(cookies, cookielib.CookieJar):

131

self.cookies = cookies

132

else:

133

self.cookies = cookiejar_from_dict(cookies)

130

134

131

135

#: True if Request has been sent.

132

136

self.sent = False

193

197

# Set encoding.

194

198

response.encoding = get_encoding_from_headers(response.headers)

195

199

196

# Start off with our local cookies.

197

cookies = self.cookies or dict()

198

199

200

# Add new cookies from the server.

200

if 'set-cookie' in response.headers:

201

cookie_header = response.headers['set-cookie']

202

cookies = dict_from_string(cookie_header)

201

extract_cookies_to_jar(self.cookies, self, resp)

203

202

204

203

# Save cookies in Response.

205

response.cookies = cookies

204

response.cookies = self.cookies

206

205

207

206

# No exceptions were harmed in the making of this request.

208

207

response.error = getattr(resp, 'error', None)

220

219

221

220

r = build(resp)

222

221

223

self.cookies.update(r.cookies)

224

225

222

if r.status_code in REDIRECT_STATI and not self.redirect:

226

223

227

224

while (('location' in r.headers) and

239

236

240

237

url = r.headers['location']

241

238

data = self.data

239

files = self.files

242

240

243

241

# Handle redirection without scheme (see: RFC 1808 Section 4)

244

242

if url.startswith('//'):

257

255

if r.status_code is codes.see_other:

258

256

method = 'GET'

259

257

data = None

258

files = None

260

259

else:

261

260

method = self.method

262

261

266

265

if r.status_code in (codes.moved, codes.found) and self.method == 'POST':

267

266

method = 'GET'

268

267

data = None

268

files = None

269

270

if (r.status_code == 303) and self.method != 'HEAD':

271

method = 'GET'

272

data = None

273

files = None

273

274

275

# Remove the cookie headers that were sent.

275

276

headers = self.headers

281

282

request = Request(

282

283

url=url,

283

284

headers=headers,

284

files=self.files,

285

files=files,

285

286

method=method,

286

287

params=self.session.params,

287

288

auth=self.auth,

299

300

301

request.send()

301

302

r = request.response

302

self.cookies.update(r.cookies)

303

304

r.history = history

305

306

self.response = r

307

self.response.request = self

308

self.response.cookies.update(self.cookies)

309

308

310

309

@staticmethod

311

310

def _encode_params(data):

312

311

"""Encode parameters in a piece of data.

313

312

314

If the data supplied is a dictionary, encodes each parameter in it, and

315

returns a list of tuples containing the encoded parameters, and a urlencoded

316

version of that.

317

318

Otherwise, assumes the data is already encoded appropriately, and

319

returns it twice.

313

Will successfully encode parameters when passed as a dict or a list of

314

2-tuples. Order is retained if data is a list of 2-tuples but abritrary

315

if parameters are supplied as a dict.

320

316

"""

321

317

322

318

if isinstance(data, bytes):

323

return data, data

324

325

if hasattr(data, '__iter__') and not isinstance(data, str):

326

data = dict(data)

327

328

if hasattr(data, 'items'):

319

return data

320

if isinstance(data, str):

321

return data

322

elif hasattr(data, '__iter__'):

323

try:

324

dict(data)

325

except ValueError:

326

raise ValueError('Unable to encode lists with elements that are not 2-tuples.')

327

328

params = list(data.items() if isinstance(data, dict) else data)

329

result = []

330

for k, vs in list(data.items()):

330

for k, vs in params:

331

for v in isinstance(vs, list) and vs or [vs]:

332

result.append((k.encode('utf-8') if isinstance(k, str) else k,

333

v.encode('utf-8') if isinstance(v, str) else v))

334

return result, urlencode(result, doseq=True)

332

result.append(

333

(k.encode('utf-8') if isinstance(k, str) else k,

334

v.encode('utf-8') if isinstance(v, str) else v))

335

return urlencode(result, doseq=True)

335

336

else:

336

return data, data

337

return data

337

338

def _encode_files(self,files):

339

def _encode_files(self, files):

339

340

341

if (not files) or isinstance(self.data, str):

341

return None, None

342

return None

342

343

344

try:

344

345

fields = self.data.copy()

352

353

else:

353

354

fn = guess_filename(v) or k

354

355

fp = v

356

if isinstance(fp, (bytes, str)):

357

fp = StringIO(fp)

355

358

fields.update({k: (fn, fp.read())})

356

359

357

360

(body, content_type) = encode_multipart_formdata(fields)

358

361

359

return files, (body, content_type)

362

return (body, content_type)

360

363

361

364

@property

362

365

def full_url(self):

381

384

if not path:

382

385

path = '/'

383

386

384

385

387

if is_py2:

386

388

if isinstance(scheme, str):

387

389

scheme = scheme.encode('utf-8')

398

400

399

401

url = (urlunparse([scheme, netloc, path, params, query, fragment]))

400

402

401

if self._enc_params:

403

enc_params = self._encode_params(self.params)

404

if enc_params:

402

405

if urlparse(url).query:

403

url = '%s&%s' % (url, self._enc_params)

406

url = '%s&%s' % (url, enc_params)

404

407

else:

405

url = '%s?%s' % (url, self._enc_params)

408

url = '%s?%s' % (url, enc_params)

406

409

407

410

if self.config.get('encode_uri', True):

408

411

url = requote_uri(url)

439

442

440

443

self.hooks[event].append(hook)

441

444

442

def deregister_hook(self,event,hook):

445

def deregister_hook(self, event, hook):

443

446

"""Deregister a previously registered hook.

444

447

Returns True if the hook existed, False if not.

445

448

"""

464

467

# Build the URL

465

468

url = self.full_url

466

469

470

# Pre-request hook.

471

r = dispatch_hook('pre_request', self.hooks, self)

472

self.__dict__.update(r.__dict__)

473

467

474

# Logging

468

475

if self.config.get('verbose'):

469

476

self.config.get('verbose').write('%s %s %s\n' % (

474

481

body = None

475

482

content_type = None

476

483

477

# Multi-part file uploads.

478

if self.files:

479

(body, content_type) = self._enc_files

480

else:

481

if self.data:

482

483

body = self._enc_data

484

if isinstance(self.data, str):

485

content_type = None

486

else:

487

content_type = 'application/x-www-form-urlencoded'

488

489

# Add content-type if it wasn't explicitly provided.

490

if (content_type) and (not 'content-type' in self.headers):

491

self.headers['Content-Type'] = content_type

492

493

484

# Use .netrc auth if none was provided.

494

485

if not self.auth and self.config.get('trust_env'):

495

486

self.auth = get_netrc_auth(url)

505

496

# Update self to reflect the auth changes.

506

497

self.__dict__.update(r.__dict__)

507

498

499

# Multi-part file uploads.

500

if self.files:

501

(body, content_type) = self._encode_files(self.files)

502

else:

503

if self.data:

504

505

body = self._encode_params(self.data)

506

if isinstance(self.data, str):

507

content_type = None

508

else:

509

content_type = 'application/x-www-form-urlencoded'

510

511

# Add content-type if it wasn't explicitly provided.

512

if (content_type) and (not 'content-type' in self.headers):

513

self.headers['Content-Type'] = content_type

514

508

515

_p = urlparse(url)

509

516

proxy = self.proxies.get(_p.scheme)

510

517

523

530

conn = self._poolmanager.connection_from_url(url)

524

531

else:

525

532

conn = connectionpool.connection_from_url(url)

533

self.headers['Connection'] = 'close'

526

534

except LocationParseError as e:

527

535

raise InvalidURL(e)

528

536

563

571

564

572

if not self.sent or anyway:

565

573

566

if self.cookies:

567

568

# Skip if 'cookie' header is explicitly set.

569

if 'cookie' not in self.headers:

570

571

# Simple cookie with our dict.

572

c = SimpleCookie()

573

for (k, v) in list(self.cookies.items()):

574

c[k] = v

575

576

# Turn it into a header.

577

cookie_header = c.output(header='', sep='; ').strip()

578

579

# Attach Cookie header to request.

574

# Skip if 'cookie' header is explicitly set.

575

if 'cookie' not in self.headers:

576

cookie_header = get_cookie_header(self.cookies, self)

577

if cookie_header is not None:

580

578

self.headers['Cookie'] = cookie_header

581

579

582

# Pre-request hook.

583

r = dispatch_hook('pre_request', self.hooks, self)

580

# Pre-send hook.

581

r = dispatch_hook('pre_send', self.hooks, self)

584

582

self.__dict__.update(r.__dict__)

585

583

586

584

try:

621

619

else:

622

620

raise

623

621

624

self._build_response(r)

622

# build_response can throw TooManyRedirects

623

try:

624

self._build_response(r)

625

except RequestException as e:

626

if self.config.get('safe_mode', False):

627

# In safe mode, catch the exception

628

self.response.error = e

629

else:

630

raise

625

631

626

632

# Response manipulation hook.

627

633

self.response = dispatch_hook('response', self.hooks, self.response)

681

687

#: The :class:`Request <Request>` that created the Response.

682

688

self.request = None

683

689

684

#: A dictionary of Cookies the server sent back.

685

self.cookies = {}

690

#: A CookieJar of Cookies the server sent back.

691

self.cookies = None

686

692

687

693

#: Dictionary of configurations for this request.

688

694

self.config = {}

748

754

chunk = pending + chunk

749

755

lines = chunk.splitlines()

750

756

751

if lines[-1][-1] == chunk[-1]:

757

if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:

752

758

pending = lines.pop()

753

759

else:

754

760

pending = None

773

779

if self.status_code is 0:

774

780

self._content = None

775

781

else:

776

self._content = bytes().join(self.iter_content()) or bytes()

782

self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()

777

783

778

784

except AttributeError:

779

785

self._content = None

781

787

self._content_consumed = True

782

788

return self._content

783

789

784

def _detected_encoding(self):

785

try:

786

detected = chardet.detect(self.content) or {}

787

return detected.get('encoding')

788

789

# Trust that chardet isn't available or something went terribly wrong.

790

except Exception:

791

pass

792

793

794

790

@property

795

791

def text(self):

796

792

"""Content of the response, in unicode.

803

799

content = None

804

800

encoding = self.encoding

805

801

806

# Fallback to auto-detected encoding if chardet is available.

802

# Fallback to auto-detected encoding.

807

803

if self.encoding is None:

808

encoding = self._detected_encoding()

804

if chardet is not None:

805

encoding = chardet.detect(self.content)['encoding']

809

806

810

807

# Decode unicode from given encoding.

811

808

try:

816

813

#

817

814

# So we try blindly encoding.

818

815

content = str(self.content, errors='replace')

819

except (UnicodeError, TypeError):

820

pass

821

816

822

817

return content

823

818

819

@property

820

def json(self):

821

"""Returns the json-encoded content of a request, if any."""

822

try:

823

return json.loads(self.text or self.content)

824

except ValueError:

825

return None

826

824

827

def raise_for_status(self, allow_redirects=True):

825

828

"""Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""

826

829

837

840

http_error.response = self

838

841

raise http_error

839

842

840

841

843

elif (self.status_code >= 500) and (self.status_code < 600):

842

844

http_error = HTTPError('%s Server Error' % self.status_code)

843

845

http_error.response = self