« back to all changes in this revision
Viewing changes to bzrlib/tests/test_http_response.py
- browse files at revision 2456
- compare with another revision
- download diff
- download tarball
- view history from revision 2456
- Committer: Tobias Toedter
- Date: 2007-12-30 18:52:13 UTC
- mfrom: (2438.1.708 +trunk)
- Revision ID: t.toedter@gmx.net-20071230185213-7xiqpbtshmnsf073
Merge trunk
- files added:
- bzrlib/help_topics
- bzrlib/help_topics/en
- files removed:
- doc/en/user-reference/index.txt
- files renamed:
- bzrlib/help_topics.py => bzrlib/help_topics/__init__.py
- doc/en/user-guide/authentication_conf.txt => bzrlib/help_topics/en/authentication.txt
- doc/en/user-guide/configuration.txt => bzrlib/help_topics/en/configuration.txt
- doc/en/user-guide/conflicts.txt => bzrlib/help_topics/en/conflicts.txt
- doc/en/user-reference/hooks.txt => bzrlib/help_topics/en/hooks.txt
- files modified:
- .bzrignore
added
removed
bzrlib/tests/test_http_response.py
1
# Copyright (C) 2005, 2006 Canonical Ltd
1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
16
17
"""Tests from HTTP response parsing."""
17
"""Tests from HTTP response parsing.
18
19
The handle_response method read the response body of a GET request an returns
20
the corresponding RangeFile.
21
22
There are four different kinds of RangeFile:
23
- a whole file whose size is unknown, seen as a simple byte stream,
24
- a whole file whose size is known, we can't read past its end,
25
- a single range file, a part of a file with a start and a size,
26
- a multiple range file, several consecutive parts with known start offset
27
and size.
28
29
Some properties are common to all kinds:
30
- seek can only be forward (its really a socket underneath),
31
- read can't cross ranges,
32
- successive ranges are taken into account transparently,
33
34
- the expected pattern of use is either seek(offset)+read(size) or a single
35
read with no size specified. For multiple range files, multiple read() will
36
return the corresponding ranges, trying to read further will raise
37
InvalidHttpResponse.
38
"""
18
39
19
40
from cStringIO import StringIO
20
import mimetools
21
22
from bzrlib import errors
23
from bzrlib.transport import http
24
from bzrlib.transport.http import response
25
from bzrlib.tests import TestCase
26
27
28
class TestResponseRange(TestCase):
29
"""Test the ResponseRange class."""
30
31
def test_cmp(self):
32
RR = response.ResponseRange
33
r1 = RR(0, 10, 0)
34
r2 = RR(15, 20, 10)
35
self.assertTrue(r1 < r2)
36
self.assertFalse(r1 > r2)
37
self.assertTrue(r1 < 5)
38
self.assertFalse(r2 < 5)
39
40
self.assertEqual(RR(0, 10, 5), RR(0, 10, 5))
41
self.assertNotEqual(RR(0, 10, 5), RR(0, 8, 5))
42
self.assertNotEqual(RR(0, 10, 5), RR(0, 10, 6))
43
44
def test_sort_list(self):
45
"""Ensure longer ranges are sorted after shorter ones"""
46
RR = response.ResponseRange
47
lst = [RR(3, 8, 0), 5, RR(3, 7, 0), 6]
48
lst.sort()
49
self.assertEqual([RR(3,7,0), RR(3,8,0), 5, 6], lst)
50
51
52
class TestRangeFile(TestCase):
53
"""Test RangeFile."""
54
55
def setUp(self):
56
content = "abcdefghijklmnopqrstuvwxyz"
57
self.fp = response.RangeFile('foo', StringIO(content))
58
self.fp._add_range(0, 9, 0)
59
self.fp._add_range(20, 29, 10)
60
self.fp._add_range(30, 39, 15)
61
62
def test_valid_accesses(self):
63
"""Test so that valid accesses work to the file."""
64
self.fp.seek(0, 0)
65
self.assertEquals(self.fp.read(3), 'abc')
66
self.assertEquals(self.fp.read(3), 'def')
67
self.assertEquals(self.fp.tell(), 6)
68
self.fp.seek(20, 0)
69
self.assertEquals(self.fp.read(3), 'klm')
70
self.assertEquals(self.fp.read(2), 'no')
71
self.assertEquals(self.fp.tell(), 25)
72
# should wrap over to 30-39 entity
73
self.assertEquals(self.fp.read(3), 'pqr')
74
self.fp.seek(3)
75
self.assertEquals(self.fp.read(3), 'def')
76
self.assertEquals(self.fp.tell(), 6)
77
78
def test_invalid_accesses(self):
79
"""Test so that invalid accesses trigger errors."""
80
self.fp.seek(9)
81
self.assertRaises(errors.InvalidRange, self.fp.read, 2)
82
self.fp.seek(39)
83
self.assertRaises(errors.InvalidRange, self.fp.read, 2)
84
self.fp.seek(19)
85
self.assertRaises(errors.InvalidRange, self.fp.read, 2)
86
87
def test__finish_ranges(self):
88
"""Test that after RangeFile._finish_ranges the list is sorted."""
89
self.fp._add_range(1, 2, 3)
90
self.fp._add_range(8, 9, 10)
91
self.fp._add_range(3, 4, 5)
92
93
# TODO: jam 20060706 If we switch to inserting
94
# in sorted order, remove this test
95
self.assertNotEqual(self.fp._ranges, sorted(self.fp._ranges))
96
97
self.fp._finish_ranges()
98
self.assertEqual(self.fp._ranges, sorted(self.fp._ranges))
99
100
def test_seek_and_tell(self):
101
# Check for seeking before start
102
self.fp.seek(-2, 0)
103
self.assertEqual(0, self.fp.tell())
104
105
self.fp.seek(5, 0)
106
self.assertEqual(5, self.fp.tell())
107
108
self.fp.seek(-2, 1)
109
self.assertEqual(3, self.fp.tell())
110
111
# TODO: jam 20060706 following tests will fail if this
112
# is not true, and would be difficult to debug
113
# but it is a layering violation
114
self.assertEqual(39, self.fp._len)
115
116
self.fp.seek(0, 2)
117
self.assertEqual(39, self.fp.tell())
118
119
self.fp.seek(-10, 2)
120
self.assertEqual(29, self.fp.tell())
121
122
self.assertRaises(ValueError, self.fp.seek, 0, 4)
123
self.assertRaises(ValueError, self.fp.seek, 0, -1)
124
125
126
class TestRegexes(TestCase):
127
128
def assertRegexMatches(self, groups, text):
129
"""Check that the regex matches and returns the right values"""
130
m = self.regex.match(text)
131
self.assertNotEqual(None, m, "text %s did not match regex" % (text,))
132
133
self.assertEqual(groups, m.groups())
134
135
def test_range_re(self):
136
"""Test that we match valid ranges."""
137
self.regex = response.HttpRangeResponse._CONTENT_RANGE_RE
138
self.assertRegexMatches(('bytes', '1', '10', '11'),
139
'bytes 1-10/11')
140
self.assertRegexMatches(('bytes', '1', '10', '11'),
141
'\tbytes 1-10/11 ')
142
self.assertRegexMatches(('bytes', '2123', '4242', '1231'),
143
'\tbytes 2123-4242/1231 ')
144
self.assertRegexMatches(('chars', '1', '2', '3'),
145
' chars 1-2/3')
146
147
def test_content_type_re(self):
148
self.regex = response.HttpMultipartRangeResponse._CONTENT_TYPE_RE
149
self.assertRegexMatches(('', 'xxyyzz'),
150
'multipart/byteranges; boundary = xxyyzz')
151
self.assertRegexMatches(('', 'xxyyzz'),
152
'multipart/byteranges;boundary=xxyyzz')
153
self.assertRegexMatches(('', 'xx yy zz'),
154
' multipart/byteranges ; boundary= xx yy zz ')
155
self.assertRegexMatches(('"', 'xx yy zz'),
156
' multipart/byteranges ; boundary= "xx yy zz" ')
157
self.assertEqual(None,
158
self.regex.match(
159
' multipart/byteranges ; boundary= "xx yy zz '))
160
self.assertEqual(None,
161
self.regex.match(
162
' multipart/byteranges ; boundary= xx yy zz" '))
163
self.assertEqual(None,
164
self.regex.match('multipart byteranges;boundary=xx'))
165
166
167
simple_data = """
168
--xxyyzz\r
169
foo\r
170
Content-range: bytes 1-10/20\r
171
\r
172
1234567890
173
--xxyyzz\r
174
Content-Range: bytes 21-30/20\r
175
bar\r
176
\r
177
abcdefghij
178
--xxyyzz\r
179
content-range: bytes 41-50/20\r
180
\r
181
zyxwvutsrq
182
--xxyyzz\r
183
content-range: bytes 51-60/20\r
184
\r
185
xxyyzz fbd
186
"""
187
188
189
class TestHelpers(TestCase):
190
"""Test the helper functions"""
191
192
def test__parse_range(self):
193
"""Test that _parse_range acts reasonably."""
194
content = StringIO('')
195
parse_range = response.HttpRangeResponse._parse_range
196
self.assertEqual((1,2), parse_range('bytes 1-2/3'))
197
self.assertEqual((10,20), parse_range('bytes 10-20/2'))
198
199
self.assertRaises(errors.InvalidHttpRange, parse_range, 'char 1-3/2')
200
self.assertRaises(errors.InvalidHttpRange, parse_range, 'bytes a-3/2')
201
202
try:
203
parse_range('bytes x-10/3', path='http://foo/bar')
204
except errors.InvalidHttpRange, e:
205
self.assertContainsRe(str(e), 'http://foo/bar')
206
self.assertContainsRe(str(e), 'bytes x-10/3')
207
else:
208
self.fail('Did not raise InvalidHttpRange')
209
210
def test__parse_boundary_simple(self):
211
"""Test that _parse_boundary handles Content-type properly"""
212
parse_boundary = response.HttpMultipartRangeResponse._parse_boundary
213
m = parse_boundary(' multipart/byteranges; boundary=xxyyzz')
214
self.assertNotEqual(None, m)
215
# Check that the returned regex is capable of splitting simple_data
216
matches = list(m.finditer(simple_data))
217
self.assertEqual(4, len(matches))
218
219
# match.group() should be the content-range entry
220
# and match.end() should be the start of the content
221
self.assertEqual(' bytes 1-10/20', matches[0].group(1))
222
self.assertEqual(simple_data.find('1234567890'), matches[0].end())
223
self.assertEqual(' bytes 21-30/20', matches[1].group(1))
224
self.assertEqual(simple_data.find('abcdefghij'), matches[1].end())
225
self.assertEqual(' bytes 41-50/20', matches[2].group(1))
226
self.assertEqual(simple_data.find('zyxwvutsrq'), matches[2].end())
227
self.assertEqual(' bytes 51-60/20', matches[3].group(1))
228
self.assertEqual(simple_data.find('xxyyzz fbd'), matches[3].end())
229
230
def test__parse_boundary_invalid(self):
231
parse_boundary = response.HttpMultipartRangeResponse._parse_boundary
232
try:
233
parse_boundary(' multipart/bytes;boundary=xxyyzz',
234
path='http://foo/bar')
235
except errors.InvalidHttpContentType, e:
236
self.assertContainsRe(str(e), 'http://foo/bar')
237
self.assertContainsRe(str(e), 'multipart/bytes;boundary=xxyyzz')
238
else:
239
self.fail('Did not raise InvalidHttpContentType')
240
241
242
class TestHttpRangeResponse(TestCase):
243
244
def test_smoketest(self):
245
"""A basic test that HttpRangeResponse is reasonable."""
246
content = StringIO('0123456789')
247
f = response.HttpRangeResponse('http://foo', 'bytes 1-10/9', content)
248
self.assertEqual([response.ResponseRange(1,10,0)], f._ranges)
249
41
import httplib
42
43
from bzrlib import (
44
errors,
45
tests,
46
)
47
from bzrlib.transport.http import (
48
response,
49
_urllib2_wrappers,
50
)
51
52
53
class ReadSocket(object):
54
"""A socket-like object that can be given a predefined content."""
55
56
def __init__(self, data):
57
self.readfile = StringIO(data)
58
59
def makefile(self, mode='r', bufsize=None):
60
return self.readfile
61
62
class FakeHTTPConnection(_urllib2_wrappers.HTTPConnection):
63
64
def __init__(self, sock):
65
_urllib2_wrappers.HTTPConnection.__init__(self, 'localhost')
66
# Set the socket to bypass the connection
67
self.sock = sock
68
69
def send(self, str):
70
"""Ignores the writes on the socket."""
71
pass
72
73
74
class TestHTTPConnection(tests.TestCase):
75
76
def test_cleanup_pipe(self):
77
sock = ReadSocket("""HTTP/1.1 200 OK\r
78
Content-Type: text/plain; charset=UTF-8\r
79
Content-Length: 18
80
\r
81
0123456789
82
garbage""")
83
conn = FakeHTTPConnection(sock)
84
# Simulate the request sending so that the connection will be able to
85
# read the response.
86
conn.putrequest('GET', 'http://localhost/fictious')
87
conn.endheaders()
88
# Now, get the response
89
resp = conn.getresponse()
90
# Read part of the response
91
self.assertEquals('0123456789\n', resp.read(11))
92
# Override the thresold to force the warning emission
93
conn._range_warning_thresold = 6 # There are 7 bytes pending
94
conn.cleanup_pipe()
95
self.assertContainsRe(self._get_log(keep_log_file=True),
96
'Got a 200 response when asking')
97
98
99
class TestRangeFileMixin(object):
100
"""Tests for accessing the first range in a RangeFile."""
101
102
# A simple string used to represent a file part (also called a range), in
103
# which offsets are easy to calculate for test writers. It's used as a
104
# building block with slight variations but basically 'a' is the first char
105
# of the range and 'z' is the last.
106
alpha = 'abcdefghijklmnopqrstuvwxyz'
107
108
def test_can_read_at_first_access(self):
109
"""Test that the just created file can be read."""
110
self.assertEquals(self.alpha, self._file.read())
111
112
def test_seek_read(self):
113
"""Test seek/read inside the range."""
114
f = self._file
115
start = self.first_range_start
116
# Before any use, tell() should be at the range start
117
self.assertEquals(start, f.tell())
118
cur = start # For an overall offset assertion
119
f.seek(start + 3)
120
cur += 3
121
self.assertEquals('def', f.read(3))
122
cur += len('def')
123
f.seek(4, 1)
124
cur += 4
125
self.assertEquals('klmn', f.read(4))
126
cur += len('klmn')
127
# read(0) in the middle of a range
128
self.assertEquals('', f.read(0))
129
# seek in place
130
here = f.tell()
131
f.seek(0, 1)
132
self.assertEquals(here, f.tell())
133
self.assertEquals(cur, f.tell())
134
135
def test_read_zero(self):
136
f = self._file
137
start = self.first_range_start
138
self.assertEquals('', f.read(0))
139
f.seek(10, 1)
140
self.assertEquals('', f.read(0))
141
142
def test_seek_at_range_end(self):
143
f = self._file
144
f.seek(26, 1)
145
146
def test_read_at_range_end(self):
147
"""Test read behaviour at range end."""
148
f = self._file
149
self.assertEquals(self.alpha, f.read())
150
self.assertEquals('', f.read(0))
151
self.assertRaises(errors.InvalidRange, f.read, 1)
152
153
def test_unbounded_read_after_seek(self):
154
f = self._file
155
f.seek(24, 1)
156
# Should not cross ranges
157
self.assertEquals('yz', f.read())
158
159
def test_seek_backwards(self):
160
f = self._file
161
start = self.first_range_start
162
f.seek(start)
163
f.read(12)
164
self.assertRaises(errors.InvalidRange, f.seek, start + 5)
165
166
def test_seek_outside_single_range(self):
167
f = self._file
168
if f._size == -1 or f._boundary is not None:
169
raise tests.TestNotApplicable('Needs a fully defined range')
170
# Will seek past the range and then errors out
171
self.assertRaises(errors.InvalidRange,
172
f.seek, self.first_range_start + 27)
173
174
def test_read_past_end_of_range(self):
175
f = self._file
176
if f._size == -1:
177
raise tests.TestNotApplicable("Can't check an unknown size")
178
start = self.first_range_start
179
f.seek(start + 20)
180
self.assertRaises(errors.InvalidRange, f.read, 10)
181
182
def test_seek_from_end(self):
183
"""Test seeking from the end of the file.
184
185
The semantic is unclear in case of multiple ranges. Seeking from end
186
exists only for the http transports, cannot be used if the file size is
187
unknown and is not used in bzrlib itself. This test must be (and is)
188
overridden by daughter classes.
189
190
Reading from end makes sense only when a range has been requested from
191
the end of the file (see HttpTransportBase._get() when using the
192
'tail_amount' parameter). The HTTP response can only be a whole file or
193
a single range.
194
"""
195
f = self._file
196
f.seek(-2, 2)
197
self.assertEquals('yz', f.read())
198
199
200
class TestRangeFileSizeUnknown(tests.TestCase, TestRangeFileMixin):
201
"""Test a RangeFile for a whole file whose size is not known."""
202
203
def setUp(self):
204
super(TestRangeFileSizeUnknown, self).setUp()
205
self._file = response.RangeFile('Whole_file_size_known',
206
StringIO(self.alpha))
207
# We define no range, relying on RangeFile to provide default values
208
self.first_range_start = 0 # It's the whole file
209
210
def test_seek_from_end(self):
211
"""See TestRangeFileMixin.test_seek_from_end.
212
213
The end of the file can't be determined since the size is unknown.
214
"""
215
self.assertRaises(errors.InvalidRange, self._file.seek, -1, 2)
216
217
def test_read_at_range_end(self):
218
"""Test read behaviour at range end."""
219
f = self._file
220
self.assertEquals(self.alpha, f.read())
221
self.assertEquals('', f.read(0))
222
self.assertEquals('', f.read(1))
223
224
class TestRangeFileSizeKnown(tests.TestCase, TestRangeFileMixin):
225
"""Test a RangeFile for a whole file whose size is known."""
226
227
def setUp(self):
228
super(TestRangeFileSizeKnown, self).setUp()
229
self._file = response.RangeFile('Whole_file_size_known',
230
StringIO(self.alpha))
231
self._file.set_range(0, len(self.alpha))
232
self.first_range_start = 0 # It's the whole file
233
234
235
class TestRangeFileSingleRange(tests.TestCase, TestRangeFileMixin):
236
"""Test a RangeFile for a single range."""
237
238
def setUp(self):
239
super(TestRangeFileSingleRange, self).setUp()
240
self._file = response.RangeFile('Single_range_file',
241
StringIO(self.alpha))
242
self.first_range_start = 15
243
self._file.set_range(self.first_range_start, len(self.alpha))
244
245
246
def test_read_before_range(self):
247
# This can't occur under normal circumstances, we have to force it
248
f = self._file
249
f._pos = 0 # Force an invalid pos
250
self.assertRaises(errors.InvalidRange, f.read, 2)
251
252
class TestRangeFilMultipleRanges(tests.TestCase, TestRangeFileMixin):
253
"""Test a RangeFile for multiple ranges.
254
255
The RangeFile used for the tests contains three ranges:
256
257
- at offset 25: alpha
258
- at offset 100: alpha
259
- at offset 126: alpha.upper()
260
261
The two last ranges are contiguous. This only rarely occurs (should not in
262
fact) in real uses but may lead to hard to track bugs.
263
"""
264
265
def setUp(self):
266
super(TestRangeFilMultipleRanges, self).setUp()
267
268
boundary = 'separation'
269
270
content = ''
271
self.first_range_start = 25
272
file_size = 200 # big enough to encompass all ranges
273
for (start, part) in [(self.first_range_start, self.alpha),
274
# Two contiguous ranges
275
(100, self.alpha),
276
(126, self.alpha.upper())]:
277
content += self._multipart_byterange(part, start, boundary,
278
file_size)
279
# Final boundary
280
content += self._boundary_line(boundary)
281
282
self._file = response.RangeFile('Multiple_ranges_file',
283
StringIO(content))
284
# Ranges are set by decoding the range headers, the RangeFile user is
285
# supposed to call the following before using seek or read since it
286
# requires knowing the *response* headers (in that case the boundary
287
# which is part of the Content-Type header).
288
self._file.set_boundary(boundary)
289
290
def _boundary_line(self, boundary):
291
"""Helper to build the formatted boundary line."""
292
return '--' + boundary + '\r\n'
293
294
def _multipart_byterange(self, data, offset, boundary, file_size='*'):
295
"""Encode a part of a file as a multipart/byterange MIME type.
296
297
When a range request is issued, the HTTP response body can be
298
decomposed in parts, each one representing a range (start, size) in a
299
file.
300
301
:param data: The payload.
302
:param offset: where data starts in the file
303
:param boundary: used to separate the parts
304
:param file_size: the size of the file containing the range (default to
305
'*' meaning unknown)
306
307
:return: a string containing the data encoded as it will appear in the
308
HTTP response body.
309
"""
310
bline = self._boundary_line(boundary)
311
# Each range begins with a boundary line
312
range = bline
313
# A range is described by a set of headers, but only 'Content-Range' is
314
# required for our implementation (TestHandleResponse below will
315
# exercise ranges with multiple or missing headers')
316
range += 'Content-Range: bytes %d-%d/%d\r\n' % (offset,
317
offset+len(data)-1,
318
file_size)
319
range += '\r\n'
320
# Finally the raw bytes
321
range += data
322
return range
323
324
def test_read_all_ranges(self):
325
f = self._file
326
self.assertEquals(self.alpha, f.read()) # Read first range
327
f.seek(100) # Trigger the second range recognition
328
self.assertEquals(self.alpha, f.read()) # Read second range
329
self.assertEquals(126, f.tell())
330
f.seek(126) # Start of third range which is also the current pos !
331
self.assertEquals('A', f.read(1))
332
f.seek(10, 1)
333
self.assertEquals('LMN', f.read(3))
334
335
def test_seek_from_end(self):
336
"""See TestRangeFileMixin.test_seek_from_end."""
337
# The actual implementation will seek from end for the first range only
338
# and then fail. Since seeking from end is intended to be used for a
339
# single range only anyway, this test just document the actual
340
# behaviour.
341
f = self._file
342
f.seek(-2, 2)
343
self.assertEquals('yz', f.read())
344
self.assertRaises(errors.InvalidRange, f.seek, -2, 2)
345
346
def test_seek_into_void(self):
347
f = self._file
348
start = self.first_range_start
349
f.seek(start)
350
# Seeking to a point between two ranges is possible (only once) but
351
# reading there is forbidden
352
f.seek(start + 40)
353
# We crossed a range boundary, so now the file is positioned at the
354
# start of the new range (i.e. trying to seek below 100 will error out)
355
f.seek(100)
356
f.seek(125)
357
358
def test_seek_across_ranges(self):
359
f = self._file
360
start = self.first_range_start
361
f.seek(126) # skip the two first ranges
362
self.assertEquals('AB', f.read(2))
363
364
def test_seek_twice_between_ranges(self):
365
f = self._file
366
start = self.first_range_start
367
f.seek(start + 40) # Past the first range but before the second
368
# Now the file is positioned at the second range start (100)
369
self.assertRaises(errors.InvalidRange, f.seek, start + 41)
370
371
def test_seek_at_range_end(self):
372
"""Test seek behavior at range end."""
373
f = self._file
374
f.seek(25 + 25)
375
f.seek(100 + 25)
376
f.seek(126 + 25)
377
378
def test_read_at_range_end(self):
379
f = self._file
380
self.assertEquals(self.alpha, f.read())
381
self.assertEquals(self.alpha, f.read())
382
self.assertEquals(self.alpha.upper(), f.read())
383
self.assertRaises(errors.InvalidHttpResponse, f.read, 1)
384
385
386
class TestRangeFileVarious(tests.TestCase):
387
"""Tests RangeFile aspects not covered elsewhere."""
388
389
def test_seek_whence(self):
390
"""Test the seek whence parameter values."""
391
f = response.RangeFile('foo', StringIO('abc'))
392
f.set_range(0, 3)
250
393
f.seek(0)
251
self.assertRaises(errors.InvalidRange, f.read, 2)
252
f.seek(1)
253
self.assertEqual('012345', f.read(6))
254
255
def test_invalid(self):
256
try:
257
f = response.HttpRangeResponse('http://foo', 'bytes x-10/9',
258
StringIO('0123456789'))
259
except errors.InvalidHttpRange, e:
260
self.assertContainsRe(str(e), 'http://foo')
261
self.assertContainsRe(str(e), 'bytes x-10/9')
262
else:
263
self.fail('Failed to raise InvalidHttpRange')
264
265
266
class TestHttpMultipartRangeResponse(TestCase):
267
"""Test the handling of multipart range responses"""
268
269
def test_simple(self):
270
content = StringIO(simple_data)
271
multi = response.HttpMultipartRangeResponse('http://foo',
272
'multipart/byteranges; boundary = xxyyzz', content)
273
274
self.assertEqual(4, len(multi._ranges))
275
276
multi.seek(1)
277
self.assertEqual('1234567890', multi.read(10))
278
multi.seek(21)
279
self.assertEqual('abcdefghij', multi.read(10))
280
multi.seek(41)
281
self.assertEqual('zyxwvutsrq', multi.read(10))
282
multi.seek(51)
283
self.assertEqual('xxyyzz fbd', multi.read(10))
284
# TODO: jam 20060706 Currently RangeFile does not support
285
# reading across ranges. Consider adding it.
286
multi.seek(41)
287
# self.assertEqual('zyxwvutsrqxxyyzz fbd', multi.read(20))
288
self.assertRaises(errors.InvalidRange, multi.read, 20)
289
290
multi.seek(21)
291
self.assertRaises(errors.InvalidRange, multi.read, 11)
292
multi.seek(31)
293
self.assertRaises(errors.InvalidRange, multi.read, 10)
294
295
def test_invalid(self):
296
content = StringIO('')
297
try:
298
response.HttpMultipartRangeResponse('http://foo',
299
'multipart/byte;boundary=invalid', content)
300
except errors.InvalidHttpContentType, e:
301
self.assertContainsRe(str(e), 'http://foo')
302
self.assertContainsRe(str(e), 'multipart/byte;')
394
f.seek(1, 1)
395
f.seek(-1, 2)
396
self.assertRaises(ValueError, f.seek, 0, 14)
397
398
def test_range_syntax(self):
399
"""Test the Content-Range scanning."""
400
401
f = response.RangeFile('foo', StringIO())
402
403
def ok(expected, header_value):
404
f.set_range_from_header(header_value)
405
# Slightly peek under the covers to get the size
406
self.assertEquals(expected, (f.tell(), f._size))
407
408
ok((1, 10), 'bytes 1-10/11')
409
ok((1, 10), 'bytes 1-10/*')
410
ok((12, 2), '\tbytes 12-13/*')
411
ok((28, 1), ' bytes 28-28/*')
412
ok((2123, 2120), 'bytes 2123-4242/12310')
413
ok((1, 10), 'bytes 1-10/ttt') # We don't check total (ttt)
414
415
def nok(header_value):
416
self.assertRaises(errors.InvalidHttpRange,
417
f.set_range_from_header, header_value)
418
419
nok('bytes 10-2/3')
420
nok('chars 1-2/3')
421
nok('bytes xx-yyy/zzz')
422
nok('bytes xx-12/zzz')
423
nok('bytes 11-yy/zzz')
424
nok('bytes10-2/3')
303
425
304
426
305
427
# Taken from real request responses
317
439
""")
318
440
319
441
320
_missing_response = (404, """HTTP/1.1 404 Not Found\r
321
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
322
Server: Apache/2.0.54 (Fedora)\r
323
Content-Length: 336\r
324
Connection: close\r
325
Content-Type: text/html; charset=iso-8859-1\r
326
\r
327
""", """<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
328
<html><head>
329
<title>404 Not Found</title>
330
</head><body>
331
<h1>Not Found</h1>
332
<p>The requested URL /branches/bzr/jam-integration/.bzr/repository/format was not found on this server.</p>
333
<hr>
334
<address>Apache/2.0.54 (Fedora) Server at bzr.arbash-meinel.com Port 80</address>
335
</body></html>
336
""")
337
338
339
442
_single_range_response = (206, """HTTP/1.1 206 Partial Content\r
340
443
Date: Tue, 11 Jul 2006 04:45:22 GMT\r
341
444
Server: Apache/2.0.54 (Fedora)\r
414
517
--418470f848b63279b--\r
415
518
""")
416
519
520
417
521
_multipart_squid_range_response = (206, """HTTP/1.0 206 Partial Content\r
418
522
Date: Thu, 31 Aug 2006 21:16:22 GMT\r
419
523
Server: Apache/2.2.2 (Unix) DAV/2\r
446
550
""")
447
551
448
552
449
_redirect_response = (206, """HTTP/1.1 301 Moved Permanently\r
450
Date: Tue, 18 Jul 2006 20:29:22 GMT\r
451
Server: Apache/2.0.54 (Ubuntu) PHP/4.4.0-3ubuntu1 mod_ssl/2.0.54 OpenSSL/0.9.7g\r
452
Location: http://bazaar-vcs.org/bzr/bzr.dev/.bzr/repository/inventory.knit\r
453
Content-Length: 272\r
454
Keep-Alive: timeout=15, max=100\r
455
Connection: Keep-Alive\r
456
Content-Type: text/html; charset=iso-8859-1\r
457
\r
458
HTTP/1.1 206 Partial Content\r
459
Date: Tue, 18 Jul 2006 20:29:23 GMT\r
460
Server: Apache/2.0.54 (Ubuntu) PHP/4.4.0-3ubuntu1 mod_ssl/2.0.54 OpenSSL/0.9.7g\r
461
Last-Modified: Tue, 18 Jul 2006 20:24:59 GMT\r
462
ETag: "be8213-83958c-f0d3dcc0"\r
463
Accept-Ranges: bytes\r
464
Content-Length: 425\r
465
Content-Range: bytes 8623075-8623499/8623500\r
466
Keep-Alive: timeout=15, max=100\r
467
Connection: Keep-Alive\r
468
Content-Type: text/plain; charset=UTF-8\r
469
\r
470
""", """this data intentionally removed,
471
this is not meant to be tested by
472
handle_response, just _extract_headers
473
""")
474
475
476
553
# This is made up
554
_full_text_response_no_content_type = (200, """HTTP/1.1 200 OK\r
555
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
556
Server: Apache/2.0.54 (Fedora)\r
557
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
558
ETag: "56691-23-38e9ae00"\r
559
Accept-Ranges: bytes\r
560
Content-Length: 35\r
561
Connection: close\r
562
\r
563
""", """Bazaar-NG meta directory, format 1
564
""")
565
566
567
_full_text_response_no_content_length = (200, """HTTP/1.1 200 OK\r
568
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
569
Server: Apache/2.0.54 (Fedora)\r
570
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
571
ETag: "56691-23-38e9ae00"\r
572
Accept-Ranges: bytes\r
573
Connection: close\r
574
Content-Type: text/plain; charset=UTF-8\r
575
\r
576
""", """Bazaar-NG meta directory, format 1
577
""")
578
579
580
_single_range_no_content_range = (206, """HTTP/1.1 206 Partial Content\r
581
Date: Tue, 11 Jul 2006 04:45:22 GMT\r
582
Server: Apache/2.0.54 (Fedora)\r
583
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
584
ETag: "238a3c-16ec2-805c5540"\r
585
Accept-Ranges: bytes\r
586
Content-Length: 100\r
587
Connection: close\r
588
\r
589
""", """mbp@sourcefrog.net-20050309040815-13242001617e4a06
590
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e762""")
591
592
593
_single_range_response_truncated = (206, """HTTP/1.1 206 Partial Content\r
594
Date: Tue, 11 Jul 2006 04:45:22 GMT\r
595
Server: Apache/2.0.54 (Fedora)\r
596
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
597
ETag: "238a3c-16ec2-805c5540"\r
598
Accept-Ranges: bytes\r
599
Content-Length: 100\r
600
Content-Range: bytes 100-199/93890\r
601
Connection: close\r
602
Content-Type: text/plain; charset=UTF-8\r
603
\r
604
""", """mbp@sourcefrog.net-20050309040815-13242001617e4a06""")
605
606
477
607
_invalid_response = (444, """HTTP/1.1 444 Bad Response\r
478
608
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
479
609
Connection: close\r
490
620
""")
491
621
492
622
493
# This should be in test_http.py, but the headers we
494
# want to parse are here
495
class TestExtractHeader(TestCase):
496
497
def use_response(self, response):
498
self.headers = http._extract_headers(response[1], 'http://foo')
499
500
def check_header(self, header, value):
501
self.assertEqual(value, self.headers[header])
502
503
def test_full_text(self):
504
self.use_response(_full_text_response)
505
506
self.check_header('Date', 'Tue, 11 Jul 2006 04:32:56 GMT')
507
self.check_header('date', 'Tue, 11 Jul 2006 04:32:56 GMT')
508
self.check_header('Content-Length', '35')
509
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
510
self.check_header('content-type', 'text/plain; charset=UTF-8')
511
512
def test_missing_response(self):
513
self.use_response(_missing_response)
514
515
self.check_header('Content-Length', '336')
516
self.check_header('Content-Type', 'text/html; charset=iso-8859-1')
517
518
def test_single_range(self):
519
self.use_response(_single_range_response)
520
521
self.check_header('Content-Length', '100')
522
self.check_header('Content-Range', 'bytes 100-199/93890')
523
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
524
525
def test_single_range_no_content(self):
526
self.use_response(_single_range_no_content_type)
527
528
self.check_header('Content-Length', '100')
529
self.check_header('Content-Range', 'bytes 100-199/93890')
530
531
def test_multi_range(self):
532
self.use_response(_multipart_range_response)
533
534
self.check_header('Content-Length', '1534')
535
self.check_header('Content-Type',
536
'multipart/byteranges; boundary=418470f848b63279b')
537
538
def test_multi_squid_range(self):
539
self.use_response(_multipart_squid_range_response)
540
541
self.check_header('Content-Length', '598')
542
self.check_header('Content-Type',
543
'multipart/byteranges; '\
544
'boundary="squid/2.5.STABLE12:C99323425AD4FE26F726261FA6C24196"')
545
546
def test_redirect(self):
547
"""We default to returning the last group of headers in the file."""
548
self.use_response(_redirect_response)
549
self.check_header('Content-Range', 'bytes 8623075-8623499/8623500')
550
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
551
552
def test_empty(self):
553
self.assertRaises(errors.InvalidHttpResponse,
554
http._extract_headers, '', 'bad url')
555
556
def test_no_opening_http(self):
557
# Remove the HTTP line from the header
558
first, txt = _full_text_response[1].split('\r\n', 1)
559
self.assertRaises(errors.InvalidHttpResponse,
560
http._extract_headers, txt, 'missing HTTTP')
561
562
def test_trailing_whitespace(self):
563
# Test that we ignore bogus whitespace on the end
564
code, txt, body = _full_text_response
565
txt += '\r\n\n\n\n\n'
566
self.use_response((code, txt, body))
567
568
self.check_header('Date', 'Tue, 11 Jul 2006 04:32:56 GMT')
569
self.check_header('Content-Length', '35')
570
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
571
572
def test_trailing_non_http(self):
573
# Test that we ignore bogus stuff on the end
574
code, txt, body = _full_text_response
575
txt = txt + 'Foo: Bar\r\nBaz: Bling\r\n\r\n'
576
self.use_response((code, txt, body))
577
578
self.check_header('Date', 'Tue, 11 Jul 2006 04:32:56 GMT')
579
self.check_header('Content-Length', '35')
580
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
581
self.assertRaises(KeyError, self.headers.__getitem__, 'Foo')
582
583
def test_extra_whitespace(self):
584
# Test that we read an HTTP response, even with extra whitespace
585
code, txt, body = _redirect_response
586
# Find the second HTTP location
587
loc = txt.find('HTTP', 5)
588
txt = txt[:loc] + '\r\n\n' + txt[loc:]
589
self.use_response((code, txt, body))
590
self.check_header('Content-Range', 'bytes 8623075-8623499/8623500')
591
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
592
593
594
class TestHandleResponse(TestCase):
595
623
_multipart_no_content_range = (206, """HTTP/1.0 206 Partial Content\r
624
Content-Type: multipart/byteranges; boundary=THIS_SEPARATES\r
625
Content-Length: 598\r
626
\r
627
""",
628
"""\r
629
--THIS_SEPARATES\r
630
Content-Type: text/plain\r
631
\r
632
# bzr knit index 8
633
--THIS_SEPARATES\r
634
""")
635
636
637
_multipart_no_boundary = (206, """HTTP/1.0 206 Partial Content\r
638
Content-Type: multipart/byteranges; boundary=THIS_SEPARATES\r
639
Content-Length: 598\r
640
\r
641
""",
642
"""\r
643
--THIS_SEPARATES\r
644
Content-Type: text/plain\r
645
Content-Range: bytes 0-18/18672\r
646
\r
647
# bzr knit index 8
648
649
The range ended at the line above, this text is garbage instead of a boundary
650
line
651
""")
652
653
654
class TestHandleResponse(tests.TestCase):
655
656
def _build_HTTPMessage(self, raw_headers):
657
status_and_headers = StringIO(raw_headers)
658
# Get rid of the status line
659
status_and_headers.readline()
660
msg = httplib.HTTPMessage(status_and_headers)
661
return msg
662
596
663
def get_response(self, a_response):
597
664
"""Process a supplied response, and return the result."""
598
headers = http._extract_headers(a_response[1], 'http://foo')
599
return response.handle_response('http://foo', a_response[0], headers,
665
code, raw_headers, body = a_response
666
msg = self._build_HTTPMessage(raw_headers)
667
return response.handle_response('http://foo', code, msg,
600
668
StringIO(a_response[2]))
601
669
602
670
def test_full_text(self):
604
672
# It is a StringIO from the original data
605
673
self.assertEqual(_full_text_response[2], out.read())
606
674
607
def test_missing_response(self):
608
self.assertRaises(errors.NoSuchFile,
609
self.get_response, _missing_response)
610
611
675
def test_single_range(self):
612
676
out = self.get_response(_single_range_response)
613
self.assertIsInstance(out, response.HttpRangeResponse)
614
615
self.assertRaises(errors.InvalidRange, out.read, 20)
616
677
617
678
out.seek(100)
618
679
self.assertEqual(_single_range_response[2], out.read(100))
619
680
620
681
def test_single_range_no_content(self):
621
682
out = self.get_response(_single_range_no_content_type)
622
self.assertIsInstance(out, response.HttpRangeResponse)
623
624
self.assertRaises(errors.InvalidRange, out.read, 20)
625
683
626
684
out.seek(100)
627
685
self.assertEqual(_single_range_no_content_type[2], out.read(100))
628
686
687
def test_single_range_truncated(self):
688
out = self.get_response(_single_range_response_truncated)
689
# Content-Range declares 100 but only 51 present
690
self.assertRaises(errors.ShortReadvError, out.seek, out.tell() + 51)
691
629
692
def test_multi_range(self):
630
693
out = self.get_response(_multipart_range_response)
631
self.assertIsInstance(out, response.HttpMultipartRangeResponse)
632
694
633
695
# Just make sure we can read the right contents
634
696
out.seek(0)
639
701
640
702
def test_multi_squid_range(self):
641
703
out = self.get_response(_multipart_squid_range_response)
642
self.assertIsInstance(out, response.HttpMultipartRangeResponse)
643
704
644
705
# Just make sure we can read the right contents
645
706
out.seek(0)
650
711
651
712
def test_invalid_response(self):
652
713
self.assertRaises(errors.InvalidHttpResponse,
653
self.get_response, _invalid_response)
714
self.get_response, _invalid_response)
654
715
655
716
def test_full_text_no_content_type(self):
656
717
# We should not require Content-Type for a full response
657
a_response = _full_text_response
658
headers = http._extract_headers(a_response[1], 'http://foo')
659
del headers['Content-Type']
660
out = response.handle_response('http://foo', a_response[0], headers,
661
StringIO(a_response[2]))
662
self.assertEqual(_full_text_response[2], out.read())
718
code, raw_headers, body = _full_text_response_no_content_type
719
msg = self._build_HTTPMessage(raw_headers)
720
out = response.handle_response('http://foo', code, msg, StringIO(body))
721
self.assertEqual(body, out.read())
663
722
664
def test_missing_no_content_type(self):
665
# Without Content-Type we should still raise NoSuchFile on a 404
666
a_response = _missing_response
667
headers = http._extract_headers(a_response[1], 'http://missing')
668
del headers['Content-Type']
669
self.assertRaises(errors.NoSuchFile,
670
response.handle_response, 'http://missing', a_response[0], headers,
671
StringIO(a_response[2]))
723
def test_full_text_no_content_length(self):
724
code, raw_headers, body = _full_text_response_no_content_length
725
msg = self._build_HTTPMessage(raw_headers)
726
out = response.handle_response('http://foo', code, msg, StringIO(body))
727
self.assertEqual(body, out.read())
672
728
673
729
def test_missing_content_range(self):
674
a_response = _single_range_response
675
headers = http._extract_headers(a_response[1], 'http://nocontent')
676
del headers['Content-Range']
677
self.assertRaises(errors.InvalidHttpResponse,
678
response.handle_response, 'http://nocontent', a_response[0],
679
headers, StringIO(a_response[2]))
730
code, raw_headers, body = _single_range_no_content_range
731
msg = self._build_HTTPMessage(raw_headers)
732
self.assertRaises(errors.InvalidHttpResponse,
733
response.handle_response,
734
'http://bogus', code, msg, StringIO(body))
735
736
def test_multipart_no_content_range(self):
737
code, raw_headers, body = _multipart_no_content_range
738
msg = self._build_HTTPMessage(raw_headers)
739
self.assertRaises(errors.InvalidHttpResponse,
740
response.handle_response,
741
'http://bogus', code, msg, StringIO(body))
742
743
def test_multipart_no_boundary(self):
744
out = self.get_response(_multipart_no_boundary)
745
out.read() # Read the whole range
746
# Fail to find the boundary line
747
self.assertRaises(errors.InvalidHttpResponse, out.seek, 1, 1)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1