8
def fetch(url, post=False, data="", headers={}, cainfo=None, curl=None):
7
from twisted.internet.threads import deferToThread
9
class FetchError(Exception):
12
class HTTPCodeError(FetchError):
14
def __init__(self, http_code, body):
15
self.http_code = http_code
19
return "Server returned HTTP code %d" % self.http_code
22
return "<HTTPCodeError http_code=%d>" % self.http_code
25
class PyCurlError(FetchError):
26
def __init__(self, error_code, message):
27
self.error_code = error_code
28
self.message = message
31
return "Error %d: %s" % (self.error_code, self.message)
34
return "<PyCurlError args=(%d, '%s')>" % (self.error_code,
37
def fetch(url, post=False, data="", headers={}, cainfo=None, curl=None,
38
connect_timeout=30, total_timeout=600):
9
39
"""Retrieve a URL and return the content.
11
41
@param url: The url to be fetched.
38
68
curl.setopt(pycurl.URL, url)
39
69
curl.setopt(pycurl.FOLLOWLOCATION, True)
40
70
curl.setopt(pycurl.MAXREDIRS, 5)
71
curl.setopt(pycurl.CONNECTTIMEOUT, connect_timeout)
72
curl.setopt(pycurl.LOW_SPEED_LIMIT, 1)
73
curl.setopt(pycurl.LOW_SPEED_TIME, total_timeout)
74
curl.setopt(pycurl.NOSIGNAL, 1)
41
75
curl.setopt(pycurl.WRITEFUNCTION, input.write)
44
return input.getvalue()
79
except pycurl.error, e:
80
raise PyCurlError(e.args[0], e.args[1])
82
body = input.getvalue()
84
http_code = curl.getinfo(pycurl.HTTP_CODE)
86
raise HTTPCodeError(http_code, body)
48
92
parser = OptionParser()
49
parser.add_option("--method", default="GET")
93
parser.add_option("--post", action="store_true")
50
94
parser.add_option("--data", default="")
51
95
parser.add_option("--cainfo")
52
96
options, (url,) = parser.parse_args(args)
53
print fetch(url, options.method, data=options.data, cainfo=options.cainfo)
97
print fetch(url, post=options.post, data=options.data,
98
cainfo=options.cainfo)
101
def fetch_async(*args, **kwargs):
102
return deferToThread(fetch, *args, **kwargs)
56
105
if __name__ == "__main__":