1
// Copyright 2014 Canonical Ltd.
2
// Licensed under the AGPLv3, see LICENCE file for details.
4
package imagestorage_test
15
"github.com/juju/errors"
16
gitjujutesting "github.com/juju/testing"
17
jc "github.com/juju/testing/checkers"
19
jujutxn "github.com/juju/txn"
20
txntesting "github.com/juju/txn/testing"
21
gc "gopkg.in/check.v1"
24
"github.com/juju/juju/state/imagestorage"
25
"github.com/juju/juju/testing"
28
var _ = gc.Suite(&ImageSuite{})
30
func TestPackage(t *stdtesting.T) {
34
type ImageSuite struct {
36
mongo *gitjujutesting.MgoInstance
38
storage imagestorage.Storage
39
metadataCollection *mgo.Collection
40
txnRunner jujutxn.Runner
43
func (s *ImageSuite) SetUpTest(c *gc.C) {
44
s.BaseSuite.SetUpTest(c)
45
s.mongo = &gitjujutesting.MgoInstance{}
49
s.session, err = s.mongo.Dial()
50
c.Assert(err, gc.IsNil)
51
s.storage = imagestorage.NewStorage(s.session, "my-uuid")
52
s.metadataCollection = imagestorage.MetadataCollection(s.storage)
53
s.txnRunner = jujutxn.NewRunner(jujutxn.RunnerParams{Database: s.metadataCollection.Database})
54
s.patchTransactionRunner()
57
func (s *ImageSuite) TearDownTest(c *gc.C) {
59
s.mongo.DestroyWithLog()
60
s.BaseSuite.TearDownTest(c)
62
func (s *ImageSuite) patchTransactionRunner() {
63
s.PatchValue(imagestorage.TxnRunner, func(db *mgo.Database) txn.Runner {
68
func (s *ImageSuite) TestAddImage(c *gc.C) {
69
s.testAddImage(c, "some-image")
72
func (s *ImageSuite) TestAddImageReplaces(c *gc.C) {
73
s.testAddImage(c, "abc")
74
s.testAddImage(c, "defghi")
77
func checkMetadata(c *gc.C, fromDb, metadata *imagestorage.Metadata) {
78
c.Assert(fromDb.Created.IsZero(), jc.IsFalse)
79
c.Assert(fromDb.Created.Before(time.Now()), jc.IsTrue)
80
fromDb.Created = time.Time{}
81
c.Assert(metadata, gc.DeepEquals, fromDb)
84
func checkAllMetadata(c *gc.C, fromDb []*imagestorage.Metadata, metadata ...*imagestorage.Metadata) {
85
c.Assert(len(metadata), gc.Equals, len(fromDb))
86
for i, m := range metadata {
87
checkMetadata(c, fromDb[i], m)
91
func (s *ImageSuite) testAddImage(c *gc.C, content string) {
92
var r io.Reader = bytes.NewReader([]byte(content))
93
addedMetadata := &imagestorage.Metadata{
98
Size: int64(len(content)),
99
SHA256: "hash(" + content + ")",
100
SourceURL: "http://path",
102
err := s.storage.AddImage(r, addedMetadata)
103
c.Assert(err, gc.IsNil)
105
metadata, rc, err := s.storage.Image("lxc", "trusty", "amd64")
106
c.Assert(err, gc.IsNil)
107
c.Assert(r, gc.NotNil)
109
checkMetadata(c, metadata, addedMetadata)
111
data, err := ioutil.ReadAll(rc)
112
c.Assert(err, gc.IsNil)
113
c.Assert(string(data), gc.Equals, content)
116
func (s *ImageSuite) TestImage(c *gc.C) {
117
_, _, err := s.storage.Image("lxc", "trusty", "amd64")
118
c.Assert(err, jc.Satisfies, errors.IsNotFound)
119
c.Assert(err, gc.ErrorMatches, `.* image metadata not found`)
121
s.addMetadataDoc(c, "lxc", "trusty", "amd64", 3, "hash(abc)", "path", "http://path")
122
_, _, err = s.storage.Image("lxc", "trusty", "amd64")
123
c.Assert(err, jc.Satisfies, errors.IsNotFound)
124
c.Assert(err, gc.ErrorMatches, `resource at path "buckets/my-uuid/path" not found`)
126
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
127
err = managedStorage.PutForBucket("my-uuid", "path", strings.NewReader("blah"), 4)
128
c.Assert(err, gc.IsNil)
130
metadata, r, err := s.storage.Image("lxc", "trusty", "amd64")
131
c.Assert(err, gc.IsNil)
133
checkMetadata(c, metadata, &imagestorage.Metadata{
134
ModelUUID: "my-uuid",
140
SourceURL: "http://path",
143
data, err := ioutil.ReadAll(r)
144
c.Assert(err, gc.IsNil)
145
c.Assert(string(data), gc.Equals, "blah")
148
func (s *ImageSuite) TestAddImageRemovesExisting(c *gc.C) {
149
// Add a metadata doc and a blob at a known path, then
150
// call AddImage and ensure the original blob is removed.
151
s.addMetadataDoc(c, "lxc", "trusty", "amd64", 3, "hash(abc)", "path", "http://path")
152
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
153
err := managedStorage.PutForBucket("my-uuid", "path", strings.NewReader("blah"), 4)
154
c.Assert(err, gc.IsNil)
156
addedMetadata := &imagestorage.Metadata{
157
ModelUUID: "my-uuid",
162
SHA256: "hash(xyzzzz)",
163
SourceURL: "http://path",
165
err = s.storage.AddImage(strings.NewReader("xyzzzz"), addedMetadata)
166
c.Assert(err, gc.IsNil)
168
// old blob should be gone
169
_, _, err = managedStorage.GetForBucket("my-uuid", "path")
170
c.Assert(err, jc.Satisfies, errors.IsNotFound)
172
s.assertImage(c, addedMetadata, "xyzzzz")
175
func (s *ImageSuite) TestAddImageRemovesExistingRemoveFails(c *gc.C) {
176
// Add a metadata doc and a blob at a known path, then
177
// call AddImage and ensure that AddImage attempts to remove
178
// the original blob, but does not return an error if it
180
s.addMetadataDoc(c, "lxc", "trusty", "amd64", 3, "hash(abc)", "path", "http://path")
181
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
182
err := managedStorage.PutForBucket("my-uuid", "path", strings.NewReader("blah"), 4)
183
c.Assert(err, gc.IsNil)
185
storage := imagestorage.NewStorage(s.session, "my-uuid")
186
s.PatchValue(imagestorage.GetManagedStorage, imagestorage.RemoveFailsManagedStorage)
187
addedMetadata := &imagestorage.Metadata{
188
ModelUUID: "my-uuid",
193
SHA256: "hash(xyzzzz)",
194
SourceURL: "http://path",
196
err = storage.AddImage(strings.NewReader("xyzzzz"), addedMetadata)
197
c.Assert(err, gc.IsNil)
199
// old blob should still be there
200
r, _, err := managedStorage.GetForBucket("my-uuid", "path")
201
c.Assert(err, gc.IsNil)
204
s.assertImage(c, addedMetadata, "xyzzzz")
207
type errorTransactionRunner struct {
211
func (errorTransactionRunner) Run(transactions txn.TransactionSource) error {
212
return errors.New("Run fails")
215
func (s *ImageSuite) TestAddImageRemovesBlobOnFailure(c *gc.C) {
216
storage := imagestorage.NewStorage(s.session, "my-uuid")
217
s.txnRunner = errorTransactionRunner{s.txnRunner}
218
addedMetadata := &imagestorage.Metadata{
219
ModelUUID: "my-uuid",
226
err := storage.AddImage(strings.NewReader("xyzzzz"), addedMetadata)
227
c.Assert(err, gc.ErrorMatches, "cannot store image metadata: Run fails")
230
"images/%s-%s-%s:%s", addedMetadata.Kind, addedMetadata.Series, addedMetadata.Arch, addedMetadata.SHA256)
231
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
232
_, _, err = managedStorage.GetForBucket("my-uuid", path)
233
c.Assert(err, jc.Satisfies, errors.IsNotFound)
236
func (s *ImageSuite) TestAddImageRemovesBlobOnFailureRemoveFails(c *gc.C) {
237
storage := imagestorage.NewStorage(s.session, "my-uuid")
238
s.PatchValue(imagestorage.GetManagedStorage, imagestorage.RemoveFailsManagedStorage)
239
s.txnRunner = errorTransactionRunner{s.txnRunner}
240
addedMetadata := &imagestorage.Metadata{
241
ModelUUID: "my-uuid",
248
err := storage.AddImage(strings.NewReader("xyzzzz"), addedMetadata)
249
c.Assert(err, gc.ErrorMatches, "cannot store image metadata: Run fails")
251
// blob should still be there, because the removal failed.
253
"images/%s-%s-%s:%s", addedMetadata.Kind, addedMetadata.Series, addedMetadata.Arch, addedMetadata.SHA256)
254
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
255
r, _, err := managedStorage.GetForBucket("my-uuid", path)
256
c.Assert(err, gc.IsNil)
260
func (s *ImageSuite) TestAddImageSame(c *gc.C) {
261
metadata := &imagestorage.Metadata{
262
ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "0", SourceURL: "http://path",
264
for i := 0; i < 2; i++ {
265
err := s.storage.AddImage(strings.NewReader("0"), metadata)
266
c.Assert(err, gc.IsNil)
267
s.assertImage(c, metadata, "0")
271
func (s *ImageSuite) TestAddImageAndJustMetadataExists(c *gc.C) {
272
s.addMetadataDoc(c, "lxc", "trusty", "amd64", 3, "hash(abc)", "images/lxc-trusty-amd64:hash(abc)", "http://path")
273
n, err := s.metadataCollection.Count()
274
c.Assert(err, jc.ErrorIsNil)
275
c.Assert(n, gc.Equals, 1)
276
s.testAddImage(c, "abc")
277
n, err = s.metadataCollection.Count()
278
c.Assert(err, jc.ErrorIsNil)
279
c.Assert(n, gc.Equals, 1)
282
func (s *ImageSuite) TestJustMetadataFails(c *gc.C) {
283
s.addMetadataDoc(c, "lxc", "trusty", "amd64", 3, "hash(abc)", "images/lxc-trusty-amd64:hash(abc)", "http://path")
284
_, rc, err := s.storage.Image("lxc", "trusty", "amd64")
285
c.Assert(rc, gc.IsNil)
286
c.Assert(err, gc.NotNil)
289
func (s *ImageSuite) TestAddImageConcurrent(c *gc.C) {
290
metadata0 := &imagestorage.Metadata{
291
ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "0", SourceURL: "http://path",
293
metadata1 := &imagestorage.Metadata{
294
ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "1", SourceURL: "http://path",
297
addMetadata := func() {
298
err := s.storage.AddImage(strings.NewReader("0"), metadata0)
299
c.Assert(err, gc.IsNil)
300
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
301
r, _, err := managedStorage.GetForBucket("my-uuid", "images/lxc-trusty-amd64:0")
302
c.Assert(err, gc.IsNil)
305
defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata).Check()
307
err := s.storage.AddImage(strings.NewReader("1"), metadata1)
308
c.Assert(err, gc.IsNil)
310
// Blob added in before-hook should be removed.
311
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
312
_, _, err = managedStorage.GetForBucket("my-uuid", "images/lxc-trusty-amd64:0")
313
c.Assert(err, jc.Satisfies, errors.IsNotFound)
315
s.assertImage(c, metadata1, "1")
318
func (s *ImageSuite) TestAddImageExcessiveContention(c *gc.C) {
319
metadata := []*imagestorage.Metadata{
320
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "0", SourceURL: "http://path"},
321
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "1", SourceURL: "http://path"},
322
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "2", SourceURL: "http://path"},
323
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "3", SourceURL: "http://path"},
327
addMetadata := func() {
328
err := s.storage.AddImage(strings.NewReader(metadata[i].SHA256), metadata[i])
329
c.Assert(err, gc.IsNil)
332
defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata, addMetadata, addMetadata).Check()
334
err := s.storage.AddImage(strings.NewReader(metadata[0].SHA256), metadata[0])
335
c.Assert(err, gc.ErrorMatches, "cannot store image metadata: state changing too quickly; try again soon")
337
// There should be no blobs apart from the last one added by the before-hook.
338
for _, metadata := range metadata[:3] {
339
path := fmt.Sprintf("images/%s-%s-%s:%s", metadata.Kind, metadata.Series, metadata.Arch, metadata.SHA256)
340
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
341
_, _, err = managedStorage.GetForBucket("my-uuid", path)
342
c.Assert(err, jc.Satisfies, errors.IsNotFound)
345
s.assertImage(c, metadata[3], "3")
348
func (s *ImageSuite) TestDeleteImage(c *gc.C) {
349
s.addMetadataDoc(c, "lxc", "trusty", "amd64", 3, "hash(abc)", "images/lxc-trusty-amd64:sha256", "http://lxc-trusty-amd64")
350
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
351
err := managedStorage.PutForBucket("my-uuid", "images/lxc-trusty-amd64:sha256", strings.NewReader("blah"), 4)
352
c.Assert(err, gc.IsNil)
354
_, rc, err := s.storage.Image("lxc", "trusty", "amd64")
355
c.Assert(err, gc.IsNil)
356
c.Assert(rc, gc.NotNil)
359
metadata := &imagestorage.Metadata{
360
ModelUUID: "my-uuid",
366
err = s.storage.DeleteImage(metadata)
367
c.Assert(err, gc.IsNil)
369
_, _, err = managedStorage.GetForBucket("my-uuid", "images/lxc-trusty-amd64:sha256")
370
c.Assert(err, jc.Satisfies, errors.IsNotFound)
372
_, _, err = s.storage.Image("lxc", "trusty", "amd64")
373
c.Assert(err, jc.Satisfies, errors.IsNotFound)
376
func (s *ImageSuite) TestDeleteNotExistentImage(c *gc.C) {
377
metadata := &imagestorage.Metadata{
378
ModelUUID: "my-uuid",
384
err := s.storage.DeleteImage(metadata)
385
c.Assert(err, jc.Satisfies, errors.IsNotFound)
388
func (s *ImageSuite) addMetadataDoc(c *gc.C, kind, series, arch string, size int64, checksum, path, sourceURL string) {
390
Id string `bson:"_id"`
391
ModelUUID string `bson:"modelUUID"`
392
Kind string `bson:"kind"`
393
Series string `bson:"series"`
394
Arch string `bson:"arch"`
395
Size int64 `bson:"size"`
396
SHA256 string `bson:"sha256,omitempty"`
397
Path string `bson:"path"`
398
Created time.Time `bson:"created"`
399
SourceURL string `bson:"sourceurl"`
401
Id: fmt.Sprintf("my-uuid-%s-%s-%s", kind, series, arch),
402
ModelUUID: "my-uuid",
410
SourceURL: sourceURL,
412
err := s.metadataCollection.Insert(&doc)
413
c.Assert(err, gc.IsNil)
416
func (s *ImageSuite) assertImage(c *gc.C, expected *imagestorage.Metadata, content string) {
417
metadata, r, err := s.storage.Image(expected.Kind, expected.Series, expected.Arch)
418
c.Assert(err, gc.IsNil)
420
checkMetadata(c, metadata, expected)
422
data, err := ioutil.ReadAll(r)
423
c.Assert(err, gc.IsNil)
424
c.Assert(string(data), gc.Equals, content)
427
func (s *ImageSuite) createListImageMetadata(c *gc.C) []*imagestorage.Metadata {
428
s.addMetadataDoc(c, "lxc", "trusty", "amd64", 3, "hash(abc)", "images/lxc-trusty-amd64:sha256", "http://lxc-trusty-amd64")
429
metadataLxc := &imagestorage.Metadata{
430
ModelUUID: "my-uuid",
436
SourceURL: "http://lxc-trusty-amd64",
438
s.addMetadataDoc(c, "kvm", "precise", "amd64", 4, "hash(abcd)", "images/kvm-precise-amd64:sha256", "http://kvm-precise-amd64")
439
metadataKvm := &imagestorage.Metadata{
440
ModelUUID: "my-uuid",
444
SHA256: "hash(abcd)",
446
SourceURL: "http://kvm-precise-amd64",
448
return []*imagestorage.Metadata{metadataLxc, metadataKvm}
451
func (s *ImageSuite) TestListAllImages(c *gc.C) {
452
testMetadata := s.createListImageMetadata(c)
453
metadata, err := s.storage.ListImages(imagestorage.ImageFilter{})
454
c.Assert(err, gc.IsNil)
455
checkAllMetadata(c, metadata, testMetadata...)
458
func (s *ImageSuite) TestListImagesByKind(c *gc.C) {
459
testMetadata := s.createListImageMetadata(c)
460
metadata, err := s.storage.ListImages(imagestorage.ImageFilter{Kind: "lxc"})
461
c.Assert(err, gc.IsNil)
462
checkAllMetadata(c, metadata, testMetadata[0])
465
func (s *ImageSuite) TestListImagesBySeries(c *gc.C) {
466
testMetadata := s.createListImageMetadata(c)
467
metadata, err := s.storage.ListImages(imagestorage.ImageFilter{Series: "precise"})
468
c.Assert(err, gc.IsNil)
469
checkAllMetadata(c, metadata, testMetadata[1])
472
func (s *ImageSuite) TestListImagesByArch(c *gc.C) {
473
testMetadata := s.createListImageMetadata(c)
474
metadata, err := s.storage.ListImages(imagestorage.ImageFilter{Arch: "amd64"})
475
c.Assert(err, gc.IsNil)
476
checkAllMetadata(c, metadata, testMetadata...)
479
func (s *ImageSuite) TestListImagesNoMatch(c *gc.C) {
480
metadata, err := s.storage.ListImages(imagestorage.ImageFilter{Series: "utopic"})
481
c.Assert(err, gc.IsNil)
482
checkAllMetadata(c, metadata)
485
func (s *ImageSuite) TestListImagesMultiFilter(c *gc.C) {
486
testMetadata := s.createListImageMetadata(c)
487
metadata, err := s.storage.ListImages(imagestorage.ImageFilter{Series: "trusty", Arch: "amd64"})
488
c.Assert(err, gc.IsNil)
489
checkAllMetadata(c, metadata, testMetadata[0])