1
from zope.interface import implements
3
from twisted.internet import defer, protocol
4
from twisted.protocols import basic, policies
5
from twisted.web2 import stream as stream_mod, http, http_headers, responsecode
6
from twisted.web2.channel import http as httpchan
7
from twisted.web2.channel.http import PERSIST_NO_PIPELINE, PERSIST_PIPELINE
8
from twisted.web2.client import interfaces
10
#from twisted.python.util import tracer
12
class ProtocolError(Exception):
15
class ClientRequest(object):
16
"""A class for describing an HTTP request to be sent to the server.
19
def __init__(self, method, uri, headers, stream):
21
@param method: The HTTP method to for this request, ex: 'GET', 'HEAD',
25
@param uri: The URI of the resource to request, this may be absolute or
26
relative, however the interpretation of this URI is left up to the
30
@param headers: Headers to be sent to the server. It is important to
31
note that this object does not create any implicit headers. So it
32
is up to the HTTP Client to add required headers such as 'Host'.
33
@type headers: C{dict}, L{twisted.web2.http_headers.Headers}, or
36
@param stream: Content body to send to the remote HTTP server.
37
@type stream: L{twisted.web2.stream.IByteStream}
42
if isinstance(headers, http_headers.Headers):
43
self.headers = headers
45
self.headers = http_headers.Headers(headers or {})
47
if stream is not None:
48
self.stream = stream_mod.IByteStream(stream)
53
class HTTPClientChannelRequest(httpchan.HTTPParser):
54
parseCloseAsEnd = True
55
outgoing_version = "HTTP/1.1"
61
def __init__(self, channel, request, closeAfter):
62
httpchan.HTTPParser.__init__(self, channel)
63
self.request = request
64
self.closeAfter = closeAfter
65
self.transport = self.channel.transport
66
self.responseDefer = defer.Deferred()
70
request = self.request
71
if request.method == "HEAD":
72
# No incoming data will arrive.
75
l.append('%s %s %s\r\n' % (request.method, request.uri,
76
self.outgoing_version))
77
if request.headers is not None:
78
for name, valuelist in request.headers.getAllRawHeaders():
79
for value in valuelist:
80
l.append("%s: %s\r\n" % (name, value))
82
if request.stream is not None:
83
if request.stream.length is not None:
84
l.append("%s: %s\r\n" % ('Content-Length', request.stream.length))
86
# Got a stream with no length. Send as chunked and hope, against
87
# the odds, that the server actually supports chunked uploads.
88
l.append("%s: %s\r\n" % ('Transfer-Encoding', 'chunked'))
89
self.chunkedOut = True
92
l.append("%s: %s\r\n" % ('Connection', 'close'))
94
l.append("%s: %s\r\n" % ('Connection', 'Keep-Alive'))
97
self.transport.writeSequence(l)
99
d = stream_mod.StreamProducer(request.stream).beginProducing(self)
100
d.addCallback(self._finish).addErrback(self._error)
102
def registerProducer(self, producer, streaming):
103
"""Register a producer.
105
self.transport.registerProducer(producer, streaming)
107
def unregisterProducer(self):
108
self.transport.unregisterProducer()
110
def write(self, data):
113
elif self.chunkedOut:
114
self.transport.writeSequence(("%X\r\n" % len(data), data, "\r\n"))
116
self.transport.write(data)
118
def _finish(self, x):
119
"""We are finished writing data."""
121
# write last chunk and closing CRLF
122
self.transport.write("0\r\n\r\n")
125
self.channel.requestWriteFinished(self)
128
def _error(self, err):
130
self.responseDefer.errback(err)
132
def _abortWithError(self, errcode, text):
134
self.responseDefer.errback(ProtocolError(text))
136
def connectionLost(self, reason):
140
def gotInitialLine(self, initialLine):
141
parts = initialLine.split(' ', 2)
143
# Parse the initial request line
145
self._abortWithError(responsecode.BAD_REQUEST, 'Bad response line: %s' % initialLine)
148
strversion, self.code, message = parts
151
protovers = http.parseVersion(strversion)
152
if protovers[0] != 'http':
155
self._abortWithError(responsecode.BAD_REQUEST, "Unknown protocol: %s" % strversion)
158
self.version = protovers[1:3]
160
# Ensure HTTP 0 or HTTP 1.
161
if self.version[0] != 1:
162
self._abortWithError(responsecode.HTTP_VERSION_NOT_SUPPORTED, 'Only HTTP 1.x is supported.')
165
## FIXME: Actually creates Response, function is badly named!
166
def createRequest(self):
167
self.stream = stream_mod.ProducerStream()
168
self.response = http.Response(self.code, self.inHeaders, self.stream)
169
self.stream.registerProducer(self, True)
173
## FIXME: Actually processes Response, function is badly named!
174
def processRequest(self):
175
self.responseDefer.callback(self.response)
177
def handleContentChunk(self, data):
178
self.stream.write(data)
180
def handleContentComplete(self):
184
class EmptyHTTPClientManager(object):
185
"""A dummy HTTPClientManager. It doesn't do any client management, and is
186
meant to be used only when creating an HTTPClientProtocol directly.
189
implements(interfaces.IHTTPClientManager)
191
def clientBusy(self, proto):
194
def clientIdle(self, proto):
197
def clientPipelining(self, proto):
200
def clientGone(self, proto):
204
class HTTPClientProtocol(basic.LineReceiver, policies.TimeoutMixin, object):
205
"""A HTTP 1.1 Client with request pipelining support."""
208
maxHeaderLength = 10240
210
readPersistent = PERSIST_NO_PIPELINE
212
# inputTimeOut should be pending whenever a complete request has
213
# been written but the complete response has not yet been
214
# received, and be reset every time data is received.
215
inputTimeOut = 60 * 4
217
def __init__(self, manager=None):
219
@param manager: The object this client reports it state to.
220
@type manager: L{interfaces.IHTTPClientManager}
223
self.outRequest = None
226
manager = EmptyHTTPClientManager()
227
self.manager = manager
229
def lineReceived(self, line):
230
if not self.inRequests:
231
# server sending random unrequested data.
232
self.transport.loseConnection()
235
# If not currently writing this request, set timeout
236
if self.inRequests[0] is not self.outRequest:
237
self.setTimeout(self.inputTimeOut)
241
self.inRequests[0].gotInitialLine(line)
243
self.inRequests[0].lineReceived(line)
245
def rawDataReceived(self, data):
246
if not self.inRequests:
247
print "Extra raw data!"
248
# server sending random unrequested data.
249
self.transport.loseConnection()
252
# If not currently writing this request, set timeout
253
if self.inRequests[0] is not self.outRequest:
254
self.setTimeout(self.inputTimeOut)
256
self.inRequests[0].rawDataReceived(data)
258
def submitRequest(self, request, closeAfter=True):
260
@param request: The request to send to a remote server.
261
@type request: L{ClientRequest}
263
@param closeAfter: If True the 'Connection: close' header will be sent,
264
otherwise 'Connection: keep-alive'
265
@type closeAfter: C{bool}
267
@return: L{twisted.internet.defer.Deferred}
268
@callback: L{twisted.web2.http.Response} from the server.
271
# Assert we're in a valid state to submit more
272
assert self.outRequest is None
273
assert ((self.readPersistent is PERSIST_NO_PIPELINE and not self.inRequests)
274
or self.readPersistent is PERSIST_PIPELINE)
276
self.manager.clientBusy(self)
278
self.readPersistent = False
280
self.outRequest = chanRequest = HTTPClientChannelRequest(self, request, closeAfter)
281
self.inRequests.append(chanRequest)
284
return chanRequest.responseDefer
286
def requestWriteFinished(self, request):
287
assert request is self.outRequest
289
self.outRequest = None
290
# Tell the manager if more requests can be submitted.
291
self.setTimeout(self.inputTimeOut)
292
if self.readPersistent is PERSIST_PIPELINE:
293
self.manager.clientPipelining(self)
295
def requestReadFinished(self, request):
296
assert self.inRequests[0] is request
298
del self.inRequests[0]
299
self.firstLine = True
301
if not self.inRequests:
302
if self.readPersistent:
303
self.setTimeout(None)
304
self.manager.clientIdle(self)
306
# print "No more requests, closing"
307
self.transport.loseConnection()
309
def setReadPersistent(self, persist):
310
self.readPersistent = persist
312
# Tell all requests but first to abort.
313
for request in self.inRequests[1:]:
314
request.connectionLost(None)
315
del self.inRequests[1:]
317
def connectionLost(self, reason):
318
self.readPersistent = False
319
self.setTimeout(None)
320
self.manager.clientGone(self)
321
# Tell all requests to abort.
322
for request in self.inRequests:
323
if request is not None:
324
request.connectionLost(reason)
326
#isLastRequest = tracer(isLastRequest)
327
#lineReceived = tracer(lineReceived)
328
#rawDataReceived = tracer(rawDataReceived)
329
#connectionLost = tracer(connectionLost)
330
#requestReadFinished = tracer(requestReadFinished)
331
#requestWriteFinished = tracer(requestWriteFinished)
332
#submitRequest = tracer(submitRequest)
336
from twisted.internet import reactor
337
d = protocol.ClientCreator(reactor, HTTPClientProtocol).connectTCP(host, 80)
338
def gotResp(resp, num):
340
print "DATA %s: %r" % (num, n)
342
print "DONE %s" % num
343
print "GOT RESPONSE %s: %s" % (num, resp)
344
stream_mod.readStream(resp.stream, print_).addCallback(printdone)
346
proto.submitRequest(ClientRequest("GET", "/", {'Host':host}, None)).addCallback(gotResp, 1)
347
proto.submitRequest(ClientRequest("GET", "/foo", {'Host':host}, None)).addCallback(gotResp, 2)
348
d.addCallback(sendReqs)
added
removed
twisted/web2/client/http.py
