3
# Basic integration tests
5
# Author: Facundo Batista <facundo@canonical.com>
7
# Copyright (C) 2010 Canonical
10
"""Basic Integration Tests."""
16
from os.path import join
18
from twisted.internet import defer
20
from ubuntuone.storage.tests.integration.helpers import (
21
create_file_and_add_content,
27
@defer.inlineCallbacks
28
def test_empty_file(test_name, sd1, sd2, sd3, prefix):
29
"""01. Create an empty file. Check."""
30
# create the file in sd1 and wait
31
open(join(sd1.rootdir, 'file.txt'), 'w').close()
32
debug(prefix, "File created")
33
yield sd2.sdt.wait_for_nirvana(.5)
35
# check that the file is in sd2
36
assert os.path.exists(join(sd2.rootdir, 'file.txt'))
39
@defer.inlineCallbacks
40
def test_empty_file_funny_name(test_name, sd1, sd2, sd3, prefix):
41
"""02. Create an empty file with a non-ascii name. Check."""
42
# create the file in sd1 and wait
43
open(join(sd1.rootdir, u'moño.txt'), 'w').close()
44
debug(prefix, "File created")
45
yield sd2.sdt.wait_for_nirvana(.5)
47
# check that the file is in sd2
48
assert os.path.exists(join(sd2.rootdir, u'moño.txt'))
51
@defer.inlineCallbacks
52
def test_file_with_content(test_name, sd1, sd2, sd3, prefix):
53
"""02. Create a file with data in it. Check."""
54
# create the file in sd1 and wait
55
filepath = join(sd1.rootdir, 'file.txt')
56
content = create_file_and_add_content(filepath)
57
debug(prefix, "File created")
58
yield sd2.sdt.wait_for_nirvana(.5)
60
# check that the file is in sd2 with correct content
61
data = open(join(sd2.rootdir, 'file.txt')).read()
62
assert data == content
65
@defer.inlineCallbacks
66
def test_file_with_content_changing(test_name, sd1, sd2, sd3, prefix):
67
"""03. Create a file with data. Check. Modify file's content. Check."""
68
# create the file in sd1 and wait
69
filepath = join(sd1.rootdir, 'file.txt')
70
content = create_file_and_add_content(filepath)
71
debug(prefix, "File created")
72
yield sd2.sdt.wait_for_nirvana(.5)
74
# check that the file is in sd2 with correct content
75
data = open(join(sd2.rootdir, 'file.txt')).read()
76
assert data == content
78
# change the content in sd2, and wait for both to settle
79
newcontent = os.urandom(1000)
80
with open(join(sd2.rootdir, 'file.txt'), 'w') as fh:
82
yield sd2.sdt.wait_for_nirvana(.5)
83
yield sd1.sdt.wait_for_nirvana(.5)
85
# check that the file in sd1 has the correct content
86
data = open(join(sd1.rootdir, 'file.txt')).read()
87
assert data == newcontent
90
@defer.inlineCallbacks
91
def test_rename_file(test_name, sd1, sd2, sd3, prefix):
92
"""04. Create a file. Check. Rename it. Check."""
93
# create the file in sd1 and wait
94
open(join(sd1.rootdir, 'file.txt'), 'w').close()
95
debug(prefix, "File created")
96
yield sd2.sdt.wait_for_nirvana(.5)
98
# check that the file is in sd2
99
assert os.path.exists(join(sd2.rootdir, 'file.txt'))
101
# rename the file, wait for both to settle
102
os.rename(join(sd1.rootdir, 'file.txt'), join(sd1.rootdir, 'filenew.txt'))
103
yield sd1.sdt.wait_for_nirvana(.5)
104
yield sd2.sdt.wait_for_nirvana(.5)
106
# check the new file is there, and the old one isn't
107
assert os.path.exists(join(sd2.rootdir, 'filenew.txt'))
108
assert not os.path.exists(join(sd2.rootdir, 'file.txt'))
111
@defer.inlineCallbacks
112
def test_rename_file_with_content(test_name, sd1, sd2, sd3, prefix):
113
"""05. Rename a file with content.
115
Create a file with some data, and immediately after rename it to
116
something else (it still will be in the creation/uploading process).
119
# create the file in sd1 and rename
120
content = create_file_and_add_content(join(sd1.rootdir, 'file.txt'))
121
os.rename(join(sd1.rootdir, 'file.txt'), join(sd1.rootdir, 'filenew.txt'))
122
debug(prefix, "File created and renamed")
123
yield sd2.sdt.wait_for_nirvana(.5)
125
# check that the file is in sd2 ok (and old file is not there)
126
assert not os.path.exists(join(sd2.rootdir, 'file.txt'))
127
data = open(join(sd2.rootdir, 'filenew.txt')).read()
128
assert data == content
131
@defer.inlineCallbacks
132
def test_unlink_file(test_name, sd1, sd2, sd3, prefix):
133
"""06. Create a file. Check. Unlink it. Check."""
134
# create the file in sd1 and wait
135
open(join(sd1.rootdir, 'file.txt'), 'w').close()
136
debug(prefix, "File created")
137
yield sd2.sdt.wait_for_nirvana(.5)
139
# check that the file is in sd2
140
assert os.path.exists(join(sd2.rootdir, 'file.txt'))
142
# remove the file, wait for both to settle
143
os.remove(join(sd1.rootdir, 'file.txt'))
144
yield sd1.sdt.wait_for_nirvana(.5)
145
yield sd2.sdt.wait_for_nirvana(.5)
148
assert not os.path.exists(join(sd2.rootdir, 'file.txt'))
151
@defer.inlineCallbacks
152
def test_fast_unlink_file(test_name, sd1, sd2, sd3, prefix):
153
"""07. Create a file and immediately after unlink it. Check."""
154
# create the file in sd1 and delete it
155
open(join(sd1.rootdir, 'file.txt'), 'w').close()
156
os.remove(join(sd1.rootdir, 'file.txt'))
157
debug(prefix, "File created and deleted")
160
yield sd1.sdt.wait_for_nirvana(.5)
161
yield sd2.sdt.wait_for_nirvana(.5)
162
assert not os.path.exists(join(sd1.rootdir, 'file.txt'))
163
assert not os.path.exists(join(sd2.rootdir, 'file.txt'))
166
@defer.inlineCallbacks
167
def test_create_directory(test_name, sd1, sd2, sd3, prefix):
168
"""08. Create a directory. Check."""
169
# create the dir in sd1 and wait
170
os.mkdir(join(sd1.rootdir, 'dir'))
171
debug(prefix, "Directory created")
172
yield sd2.sdt.wait_for_nirvana(.5)
174
# check that the dir is in sd2
175
assert os.path.exists(join(sd2.rootdir, 'dir'))
178
@defer.inlineCallbacks
179
def test_create_and_remove_directory(test_name, sd1, sd2, sd3, prefix):
180
"""09. Remove an empty directory. Check."""
181
# create the dir in sd1 and wait
182
os.mkdir(join(sd1.rootdir, 'dir'))
183
debug(prefix, "Directory created")
184
yield sd2.sdt.wait_for_nirvana(.5)
186
# check that the dir is in sd2
187
assert os.path.exists(join(sd2.rootdir, 'dir'))
188
debug(prefix, "Directory is ok in sd2")
191
os.rmdir(join(sd2.rootdir, 'dir'))
192
debug(prefix, "Directory removed")
193
yield sd2.sdt.wait_for_nirvana(.5)
194
yield sd1.sdt.wait_for_nirvana(.5)
196
# check it's gone also in sd1
197
assert not os.path.exists(join(sd1.rootdir, 'dir'))
200
@defer.inlineCallbacks
201
def test_create_tree_and_rename_parent_fast(test_name, sd1, sd2, sd3, prefix):
202
"""10. Create tree and rename the parent while uploading.
204
Create a directory, put three files inside with data, and immediately
205
after rename the directory. Check.
207
# create the root/a/b/c/d tree in sd1, put some files in it, rename the dir
208
deepdir = join(sd1.rootdir, *'abcd')
210
c1 = create_file_and_add_content(join(deepdir, 'file1.txt'))
211
c2 = create_file_and_add_content(join(deepdir, 'file2.txt'))
212
c3 = create_file_and_add_content(join(deepdir, 'file3.txt'))
213
os.rename(join(sd1.rootdir, 'a'), join(sd1.rootdir, 'nuevo'))
214
debug(prefix, "Tree structure created and dir removed")
216
# wait for both to settle, and check
217
yield sd1.sdt.wait_for_nirvana(.5)
218
yield sd2.sdt.wait_for_nirvana(.5)
220
data = open(join(sd2.rootdir, 'nuevo', 'b', 'c', 'd', 'file1.txt')).read()
222
data = open(join(sd2.rootdir, 'nuevo', 'b', 'c', 'd', 'file2.txt')).read()
224
data = open(join(sd2.rootdir, 'nuevo', 'b', 'c', 'd', 'file3.txt')).read()
228
@defer.inlineCallbacks
229
def test_create_both_sd_different_content(test_name, sd1, sd2, sd3, prefix):
230
"""11. Create files with different content in both SDs.
232
Create at the same time on each client two files with same name and
233
different content. Check that one of both gets conflicted.
235
# create both files with content
236
c1 = os.urandom(1000)
238
create_file_and_add_content(join(sd1.rootdir, 'file.txt'), content=c1)
239
create_file_and_add_content(join(sd2.rootdir, 'file.txt'), content=c2)
240
debug(prefix, "Files with different content created in both SDs")
242
# wait for both to settle, and get the files in both
243
yield sd1.sdt.wait_for_nirvana(.5)
244
yield sd2.sdt.wait_for_nirvana(.5)
245
files1 = walk_and_list_dir(sd1.rootdir)
246
debug(prefix, "Files in SD1", files1)
247
files2 = walk_and_list_dir(sd2.rootdir)
248
debug(prefix, "Files in SD2", files2)
250
# file.txt should be on both
251
assert "file.txt" in files1, "the file.txt should be in SD1"
252
assert "file.txt" in files2, "the file.txt should be in SD2"
254
# one of them should have a conflict, and the other should not
255
conflict1 = "file.txt.u1conflict" in files1
256
conflict2 = "file.txt.u1conflict" in files2
257
assert conflict1 and not conflict2 or conflict2 and not conflict1, \
258
"Only one of both should have conflict!"
260
@defer.inlineCallbacks
261
def test_create_both_sd_empty(test_name, sd1, sd2, sd3, prefix):
262
"""12. Create empty files in both SDs.
264
Create at the same time on each client two empty files with the same
267
# create two empty files in the SDs
268
open(join(sd1.rootdir, 'file.txt'), 'w').close()
269
open(join(sd2.rootdir, 'file.txt'), 'w').close()
270
debug(prefix, "Empty files created in both SDs")
272
# wait for both to settle, and get the files in both
273
yield sd1.sdt.wait_for_nirvana(.5)
274
yield sd2.sdt.wait_for_nirvana(.5)
275
files1 = walk_and_list_dir(sd1.rootdir)
276
files2 = walk_and_list_dir(sd2.rootdir)
278
# file.txt should be on both, and no conflict at all
279
assert "file.txt" in files1, "the file should be in SD1: %s" % (files1,)
280
assert "file.txt.u1conflict" not in files1, "conflict SD1: %s" % (files1,)
281
assert "file.txt" in files2, "the file should be in SD2: %s" % (files2,)
282
assert "file.txt.u1conflict" not in files2, "conflict SD2: %s" % (files2,)
285
@defer.inlineCallbacks
286
def test_create_both_sd_same_content(test_name, sd1, sd2, sd3, prefix):
287
"""13. Create files with same content in both SDs.
289
Create at the same time on each client two files with same name and
292
# create both files with same content
293
data = os.urandom(1000)
294
create_file_and_add_content(join(sd1.rootdir, 'file.txt'), content=data)
295
create_file_and_add_content(join(sd2.rootdir, 'file.txt'), content=data)
296
debug(prefix, "Files with same content created in both SDs")
298
# wait for both to settle, and get the files in both
299
yield sd1.sdt.wait_for_nirvana(.5)
300
yield sd2.sdt.wait_for_nirvana(.5)
301
files1 = walk_and_list_dir(sd1.rootdir)
302
files2 = walk_and_list_dir(sd2.rootdir)
304
# file.txt should be on both, and no conflict at all
305
assert "file.txt" in files1, "the file should be in SD1: %s" % (files1,)
306
assert "file.txt.u1conflict" not in files1, "conflict SD1: %s" % (files1,)
307
assert "file.txt" in files2, "the file should be in SD2: %s" % (files2,)
308
assert "file.txt.u1conflict" not in files2, "conflict SD2: %s" % (files2,)
311
@defer.inlineCallbacks
312
def test_create_both_sd_same_dir_diff_file(test_name, sd1, sd2, sd3, prefix):
313
"""14. Create different files in same dir in both SDs.
315
Create at the same time on one client a directory with file A, and on
316
the other client the same directory with file B. Check that both files
319
# create both dirs with different files
320
os.mkdir(join(sd1.rootdir, 'dir'))
321
os.mkdir(join(sd2.rootdir, 'dir'))
322
open(join(sd1.rootdir, 'dir', 'file1.txt'), 'w').close()
323
open(join(sd2.rootdir, 'dir', 'file2.txt'), 'w').close()
324
debug(prefix, "Dir and files created in both SDs")
326
# wait for both to settle, and get the files in both
327
yield sd1.sdt.wait_for_nirvana(.5)
328
yield sd2.sdt.wait_for_nirvana(.5)
329
files1 = walk_and_list_dir(join(sd1.rootdir, 'dir'))
330
files2 = walk_and_list_dir(join(sd2.rootdir, 'dir'))
332
# both files should be on both
333
assert "file1.txt" in files1, "file 1 should be in SD1: %s" % (files1,)
334
assert "file2.txt" in files1, "file 2 should be in SD1: %s" % (files1,)
335
assert "file1.txt" in files2, "file 1 should be in SD2: %s" % (files2,)
336
assert "file2.txt" in files2, "file 2 should be in SD2: %s" % (files2,)
339
@defer.inlineCallbacks
340
def test_create_tree_and_rename(test_name, sd1, sd2, sd3, prefix):
341
"""15. Create a tree and rename parent.
343
Create a directory, with one subdir, and other subdir under the later,
344
and a file as a leaf (root/a/b/c.txt). Wait for nirvana. Rename the
345
first subdir (a). Check.
347
# create the tree and wait for it to finish
348
os.makedirs(join(sd1.rootdir, 'a', 'b'))
349
open(join(sd1.rootdir, 'a', 'b', 'c.txt'), 'w').close()
350
debug(prefix, "Dir and files created in SD1")
351
yield sd1.sdt.wait_for_nirvana(.5)
353
# rename the parent, wait for both to settle, and get the files in both
354
os.rename(join(sd1.rootdir, 'a'), join(sd1.rootdir, 'nuevo'))
355
debug(prefix, "Parent renamed")
356
yield sd1.sdt.wait_for_nirvana(.5)
357
yield sd2.sdt.wait_for_nirvana(.5)
358
files1 = walk_and_list_dir(sd1.rootdir)
359
files2 = walk_and_list_dir(sd2.rootdir)
361
# both trees should be on both, and old 'a' on neither of them
362
assert "nuevo/b/c.txt" in files1, "tree not in SD1: %s" % (files1,)
363
assert "nuevo/b/c.txt" in files2, "tree not in SD2: %s" % (files2,)
364
assert 'a' not in files1, "'a' should not be in SD1: %s" % (files1,)
365
assert 'a' not in files2, "'a' should not be in SD1: %s" % (files2,)
368
@defer.inlineCallbacks
369
def test_create_tree_and_remove_it(test_name, sd1, sd2, sd3, prefix):
370
"""16. Create tree and remove it.
372
Create a tree structure with some dirs and some files in some of
373
them. Wait for nirvana. Remove everything as fast as possible. Check.
375
# create some tree structure with files
376
deepdir = join(sd1.rootdir, *'abcd')
378
create_file_and_add_content(join(deepdir, 'file1.txt'))
379
create_file_and_add_content(join(deepdir, 'file2.txt'))
380
open(join(deepdir, 'file2.txt'), 'w').close()
382
deepdir = join(sd1.rootdir, *'abjk')
384
create_file_and_add_content(join(deepdir, 'file3.txt'))
385
open(join(deepdir, 'file4.txt'), 'w').close()
387
deepdir = join(sd1.rootdir, 'a')
388
create_file_and_add_content(join(deepdir, 'file5.txt'))
389
open(join(deepdir, 'file6.txt'), 'w').close()
390
debug(prefix, "Tree structure created with files in it")
392
# wait for both to settle, and remove everything
393
yield sd1.sdt.wait_for_nirvana(.5)
394
yield sd2.sdt.wait_for_nirvana(.5)
395
shutil.rmtree(join(sd1.rootdir, 'a'))
396
debug(prefix, "rmtree finished")
398
# wait for everything to finish, get both files list and check
399
yield sd1.sdt.wait_for_nirvana(.5)
400
yield sd2.sdt.wait_for_nirvana(.5)
401
files1 = walk_and_list_dir(sd1.rootdir)
402
files2 = walk_and_list_dir(sd2.rootdir)
403
assert files1 == [], "bad info in SD1: %s" % (files1,)
404
assert files2 == [], "bad info in SD1: %s" % (files2,)
407
@defer.inlineCallbacks
408
def test_rename_dir_create_same_name(test_name, sd1, sd2, sd3, prefix):
409
"""17. Create directory, rename it and create other with same name.
411
Create a directory with a file A in it. Wait for nirvana. Rename
412
the directory and immediately after create other directory with same
413
name, and file B in it. Check.
415
# create the directory, file, and wait
416
os.mkdir(join(sd1.rootdir, 'direct'))
417
open(join(sd1.rootdir, 'direct', 'fileA.txt'), 'w').close()
418
debug(prefix, "Directory with file A created")
419
yield sd1.sdt.wait_for_nirvana(.5)
420
debug(prefix, "Nirvana reached")
422
# rename the directory, create again, and a new file
423
os.rename(join(sd1.rootdir, 'direct'), join(sd1.rootdir, 'newdir'))
424
os.mkdir(join(sd1.rootdir, 'direct'))
425
open(join(sd1.rootdir, 'direct', 'fileB.txt'), 'w').close()
426
debug(prefix, "Directory renamed and created new one with file B")
428
yield sd1.sdt.wait_for_nirvana(.5)
429
yield sd2.sdt.wait_for_nirvana(.5)
430
debug(prefix, "Nirvana reached on both")
433
files = walk_and_list_dir(sd2.rootdir)
434
shouldbe = ['direct', 'direct/fileB.txt', 'newdir', 'newdir/fileA.txt']
435
assert files == shouldbe, "Bad files in SD2: %s" % (files,)
438
@defer.inlineCallbacks
439
def test_create_tree_internal_move(test_name, sd1, sd2, sd3, prefix):
440
"""18. Create tree and do internal move.
442
Create a directory, with a subdir and a file in the later; immediately
443
after move the file one dir up. Check.
445
# create the tree and do the move
446
os.makedirs(join(sd1.rootdir, 'dir1', 'dir2'))
447
content = create_file_and_add_content(join(sd1.rootdir, 'dir1',
449
os.rename(join(sd1.rootdir, 'dir1', 'dir2', 'file.txt'),
450
join(sd1.rootdir, 'dir1', 'file.txt'))
451
debug(prefix, "Tree created and file moved")
453
yield sd1.sdt.wait_for_nirvana(.5)
454
yield sd2.sdt.wait_for_nirvana(.5)
455
debug(prefix, "Nirvana reached on both")
458
files = walk_and_list_dir(sd2.rootdir)
459
shouldbe = ['dir1', 'dir1/dir2', 'dir1/file.txt']
460
assert files == shouldbe, "Bad files in SD2: %s" % (files,)
462
# check content is ok
463
assert content == open(join(sd2.rootdir, 'dir1', 'file.txt')).read()
466
@defer.inlineCallbacks
467
def test_delete_tree_touched_ok(test_name, sd1, sd2, sd3, prefix):
468
"""19. Delete a tree on one side that has a deleted file in the other.
470
Create a directory, with a subdir and a file in the later. Wait for
471
nirvana. At the same time on one client unlink the file, and on the
472
other client remove everything. Check that all is deleted ok.
474
# create the tree and wait
475
os.makedirs(join(sd1.rootdir, 'dir1', 'dir2'))
476
content = create_file_and_add_content(join(sd1.rootdir, 'dir1',
478
yield sd1.sdt.wait_for_nirvana(.5)
479
debug(prefix, "Tree created and propagated")
481
# remove everything on one side, and unlink on the other
482
shutil.rmtree(join(sd2.rootdir, 'dir1'))
483
os.remove(join(sd1.rootdir, 'dir1', 'dir2', 'file.txt'))
484
debug(prefix, "Deletes executed")
487
yield sd1.sdt.wait_for_nirvana(.5)
488
yield sd2.sdt.wait_for_nirvana(.5)
489
debug(prefix, "Nirvana reached on both")
492
files1 = walk_and_list_dir(sd1.rootdir)
493
files2 = walk_and_list_dir(sd2.rootdir)
494
assert files1 == [], "Bad files in SD1: %s" % (files1,)
495
assert files2 == [], "Bad files in SD2: %s" % (files2,)
498
@defer.inlineCallbacks
499
def test_delete_tree_touched_conflict(test_name, sd1, sd2, sd3, prefix):
500
"""20. Delete a tree on one side that has a renamed file in the other.
502
Create a directory, with a subdir and a file in the later. Wait for
503
nirvana. Disconnect SD1 and remove everything in SD2. Write something
504
to the file in SD1 and connect again. Check that in SD2 all is gone,
505
but in SD1 the directory is not deleted but conflicted.
507
# create the tree and wait
508
os.makedirs(join(sd1.rootdir, 'dir1', 'dir2'))
509
content = create_file_and_add_content(join(sd1.rootdir, 'dir1',
511
yield sd1.sdt.wait_for_nirvana(.5)
512
yield sd2.sdt.wait_for_nirvana(.5)
513
yield sd1.sdt.disconnect()
514
debug(prefix, "Tree created and propagated; sd1 disconnected")
516
# remove everything on one side
517
shutil.rmtree(join(sd2.rootdir, 'dir1'))
518
yield sd2.sdt.wait_for_nirvana(.5)
519
debug(prefix, "Delete executed")
521
# change sd1 and reconnect
522
with open(join(sd1.rootdir, 'dir1', 'dir2', 'file.txt'), 'w') as fh:
523
fh.write(os.urandom(10))
524
yield sd1.sdt.connect()
525
yield sd1.sdt.wait_for_nirvana(.5)
526
debug(prefix, "Wrote in SD1 and reconnected")
529
files2 = walk_and_list_dir(sd2.rootdir)
530
assert files2 == [], "Bad files in SD2: %s" % (files2,)
532
files1 = walk_and_list_dir(sd1.rootdir)
533
shouldbe = ['dir1.u1conflict', 'dir1.u1conflict/dir2.u1conflict',
534
'dir1.u1conflict/dir2.u1conflict/file.txt.u1conflict']
535
assert files1 == shouldbe, "Bad files in SD1: %s" % (files1,)
538
@defer.inlineCallbacks
539
def test_overwrite_file_with_content(test_name, sd1, sd2, sd3, prefix):
540
"""21. Create a file with data in it and overwrite it.
542
Create a file 1 with content, and a file 2 with other content. Wait
543
for nirvana. Move 2 into 1 (overwriting it). Check.
545
# create the file in sd1 and wait
546
file1 = join(sd1.rootdir, 'file1.txt')
547
content1 = create_file_and_add_content(file1)
548
file2 = join(sd1.rootdir, 'file2.txt')
549
content2 = create_file_and_add_content(file2)
550
yield sd1.sdt.wait_for_nirvana(.5)
551
debug(prefix, "Files created")
554
os.rename(file2, file1)
555
debug(prefix, "Overwritten")
558
yield sd1.sdt.wait_for_nirvana(.5)
559
yield sd2.sdt.wait_for_nirvana(.5)
560
debug(prefix, "All changes propagated")
562
# check that the file is in sd2 with correct content
563
assert not os.path.exists(join(sd1.rootdir, 'file2.txt'))
564
data = open(join(sd2.rootdir, 'file1.txt')).read()
565
assert data == content2