← Back to branch summary

~philroche/simplestreams/bionic-i386-ova-not-expected

« back to all changes in this revision

Viewing changes to tests/unittests/test_glancemirror.py

GlanceMirror: refactor insert_item for easier testing

This change refactors GlanceMirror.insert_item() to allow for easier and
more contained testing. I needed to do this to understand everything that
was going on inside insert_item and other bits of code. There are now four
distinct things happening in it:

 1. Download image to a local file from a ContentSource
 2. Construct extra properties to store in Glance along with image
 3. Prepare arguments for GlanceClient.images.create() call
 4. Adapt source simplestreams entry for an image for use in the target
    simplestreams index

It should be fully backwards compatible, and test coverage for all the
individual steps should be much better (I admit to it not being perfect,
but it's a step in the right direction, imho at least).

Show diffs side-by-side

added added

removed removed

Lines of Context:
tests/unittests/test_glancemirror.py
 
1
from simplestreams.contentsource import MemoryContentSource
 
2
from simplestreams.mirrors.glance import GlanceMirror
 
3
import simplestreams.util
 
4
 
 
5
import os
 
6
from unittest import TestCase
 
7
 
 
8
 
 
9
class FakeOpenstack(object):
 
10
    """Fake 'openstack' module replacement for testing GlanceMirror."""
 
11
    def load_keystone_creds(self):
 
12
        return {"auth_url": "http://keystone/api/"}
 
13
 
 
14
    def get_service_conn_info(self, url, region_name=None, auth_url=None):
 
15
        return {"endpoint": "http://objectstore/api/",
 
16
                "tenant_id": "bar456"}
 
17
 
 
18
 
 
19
class FakeImage(object):
 
20
    """Fake image objects returned by GlanceClient.images.create()."""
 
21
    def __init__(self, identifier):
 
22
        self.id = identifier
 
23
 
 
24
 
 
25
class FakeImages(object):
 
26
    """Fake GlanceClient.images implementation to track create() calls."""
 
27
    def __init__(self):
 
28
        self.create_calls = []
 
29
 
 
30
    def create(self, **kwargs):
 
31
        self.create_calls.append(kwargs)
 
32
        return FakeImage('image-%d' % len(self.create_calls))
 
33
 
 
34
 
 
35
class FakeGlanceClient(object):
 
36
    """Fake GlanceClient implementation to track images.create() calls."""
 
37
    def __init__(self, *args):
 
38
        self.images = FakeImages()
 
39
 
 
40
 
 
41
class TestGlanceMirror(TestCase):
 
42
    """Tests for GlanceMirror methods."""
 
43
 
 
44
    def setUp(self):
 
45
        self.config = {"content_id": "foo123"}
 
46
        self.mirror = GlanceMirror(
 
47
            self.config, name_prefix="auto-sync/", region="region1",
 
48
            client=FakeOpenstack())
 
49
 
 
50
    def test_adapt_source_entry(self):
 
51
        # Adapts source entry for use in a local simplestreams index.
 
52
        source_entry = {"source-key": "source-value"}
 
53
        output_entry = self.mirror.adapt_source_entry(
 
54
            source_entry, hypervisor_mapping=False, image_name="foobuntu-X",
 
55
            image_md5_hash=None, image_size=None)
 
56
 
 
57
        # Source and output entry are different objects.
 
58
        self.assertNotEqual(source_entry, output_entry)
 
59
 
 
60
        # Output entry gets a few new properties like the endpoint and
 
61
        # owner_id taken from the GlanceMirror and  OpenStack configuration,
 
62
        # region from the value passed into GlanceMirror constructor, and
 
63
        # image name from the passed in value.
 
64
        # It also contains the source entries as well.
 
65
        self.assertEqual(
 
66
            {"endpoint": "http://keystone/api/",
 
67
             "name": "foobuntu-X",
 
68
             "owner_id": "bar456",
 
69
             "region": "region1",
 
70
             "source-key": "source-value"},
 
71
            output_entry)
 
72
 
 
73
    def test_adapt_source_entry_ignored_properties(self):
 
74
        # adapt_source_entry() drops some properties from the source entry.
 
75
        source_entry = {"path": "foo",
 
76
                        "product_name": "bar",
 
77
                        "version_name": "baz",
 
78
                        "item_name": "bah"}
 
79
        output_entry = self.mirror.adapt_source_entry(
 
80
            source_entry, hypervisor_mapping=False, image_name="foobuntu-X",
 
81
            image_md5_hash=None, image_size=None)
 
82
 
 
83
        # None of the values in 'source_entry' are preserved.
 
84
        for key in ("path", "product_name", "version_name", "item"):
 
85
            self.assertNotIn("path", output_entry)
 
86
 
 
87
    def test_adapt_source_entry_image_md5_and_size(self):
 
88
        # adapt_source_entry() will use passed in values for md5 and size.
 
89
        # Even old stale values will be overridden when image_md5_hash and
 
90
        # image_size are passed in.
 
91
        source_entry = {"md5": "stale-md5"}
 
92
        output_entry = self.mirror.adapt_source_entry(
 
93
            source_entry, hypervisor_mapping=False, image_name="foobuntu-X",
 
94
            image_md5_hash="new-md5", image_size=5)
 
95
 
 
96
        self.assertEqual("new-md5", output_entry["md5"])
 
97
        self.assertEqual(5, output_entry["size"])
 
98
 
 
99
    def test_adapt_source_entry_image_md5_and_size_both_required(self):
 
100
        # adapt_source_entry() requires both md5 and size to not ignore them.
 
101
 
 
102
        source_entry = {"md5": "stale-md5"}
 
103
 
 
104
        # image_size is not passed in, so md5 value is not used either.
 
105
        output_entry1 = self.mirror.adapt_source_entry(
 
106
            source_entry, hypervisor_mapping=False, image_name="foobuntu-X",
 
107
            image_md5_hash="new-md5", image_size=None)
 
108
        self.assertEqual("stale-md5", output_entry1["md5"])
 
109
        self.assertNotIn("size", output_entry1)
 
110
 
 
111
        # image_md5_hash is not passed in, so image_size is not used either.
 
112
        output_entry2 = self.mirror.adapt_source_entry(
 
113
            source_entry, hypervisor_mapping=False, image_name="foobuntu-X",
 
114
            image_md5_hash=None, image_size=5)
 
115
        self.assertEqual("stale-md5", output_entry2["md5"])
 
116
        self.assertNotIn("size", output_entry2)
 
117
 
 
118
    def test_adapt_source_entry_hypervisor_mapping(self):
 
119
        # If hypervisor_mapping is set to True, 'virt' value is derived from
 
120
        # the source entry 'ftype'.
 
121
        source_entry = {"ftype": "disk1.img"}
 
122
        output_entry = self.mirror.adapt_source_entry(
 
123
            source_entry, hypervisor_mapping=True, image_name="foobuntu-X",
 
124
            image_md5_hash=None, image_size=None)
 
125
 
 
126
        self.assertEqual("kvm", output_entry["virt"])
 
127
 
 
128
    def test_adapt_source_entry_hypervisor_mapping_ftype_required(self):
 
129
        # If hypervisor_mapping is set to True, but 'ftype' is missing in the
 
130
        # source entry, 'virt' value is not added to the returned entry.
 
131
        source_entry = {}
 
132
        output_entry = self.mirror.adapt_source_entry(
 
133
            source_entry, hypervisor_mapping=True, image_name="foobuntu-X",
 
134
            image_md5_hash=None, image_size=None)
 
135
 
 
136
        self.assertNotIn("virt", output_entry)
 
137
 
 
138
    def test_create_glance_properties(self):
 
139
        # Constructs glance properties to set on image during upload
 
140
        # based on source image metadata.
 
141
        source_entry = {
 
142
            # All of these are carried over and potentially re-named.
 
143
            "product_name": "foobuntu",
 
144
            "version_name": "X",
 
145
            "item_name": "disk1.img",
 
146
            "os": "ubuntu",
 
147
            "version": "16.04",
 
148
            # Other entries are ignored.
 
149
            "something-else": "ignored",
 
150
        }
 
151
        properties = self.mirror.create_glance_properties(
 
152
            "content-1", "source-1", source_entry, hypervisor_mapping=False)
 
153
 
 
154
        # Output properties contain content-id and source-content-id based
 
155
        # on the passed in parameters, and carry over (with changed keys
 
156
        # for "os" and "version") product_name, version_name, item_name and
 
157
        # os and version values from the source entry.
 
158
        self.assertEqual(
 
159
            {"content_id": "content-1",
 
160
             "source_content_id": "source-1",
 
161
             "product_name": "foobuntu",
 
162
             "version_name": "X",
 
163
             "item_name": "disk1.img",
 
164
             "os_distro": "ubuntu",
 
165
             "os_version": "16.04"},
 
166
            properties)
 
167
 
 
168
    def test_create_glance_properties_arch(self):
 
169
        # When 'arch' is present in the source entry, it is adapted and
 
170
        # returned inside 'architecture' field.
 
171
        source_entry = {
 
172
            "product_name": "foobuntu",
 
173
            "version_name": "X",
 
174
            "item_name": "disk1.img",
 
175
            "os": "ubuntu",
 
176
            "version": "16.04",
 
177
            "arch": "amd64",
 
178
        }
 
179
        properties = self.mirror.create_glance_properties(
 
180
            "content-1", "source-1", source_entry, hypervisor_mapping=False)
 
181
        self.assertEqual("x86_64", properties["architecture"])
 
182
 
 
183
    def test_create_glance_properties_hypervisor_mapping(self):
 
184
        # When hypervisor_mapping is requested and 'ftype' is present in
 
185
        # the image metadata, 'hypervisor_type' is added to returned
 
186
        # properties.
 
187
        source_entry = {
 
188
            "product_name": "foobuntu",
 
189
            "version_name": "X",
 
190
            "item_name": "disk1.img",
 
191
            "os": "ubuntu",
 
192
            "version": "16.04",
 
193
            "ftype": "root.tar.gz",
 
194
        }
 
195
        properties = self.mirror.create_glance_properties(
 
196
            "content-1", "source-1", source_entry, hypervisor_mapping=True)
 
197
        self.assertEqual("lxc", properties["hypervisor_type"])
 
198
 
 
199
    def test_prepare_glance_arguments(self):
 
200
        # Prepares arguments to pass to GlanceClient.images.create()
 
201
        # based on image metadata from the simplestreams source.
 
202
        source_entry = {}
 
203
        create_arguments = self.mirror.prepare_glance_arguments(
 
204
            "foobuntu-X", source_entry, image_md5_hash=None, image_size=None,
 
205
            image_properties=None)
 
206
 
 
207
        # Arguments to always pass in contain the image name, container format,
 
208
        # disk format, whether image is public, and any passed-in properties.
 
209
        self.assertEqual(
 
210
            {"name": "foobuntu-X",
 
211
             "container_format": 'bare',
 
212
             "disk_format": "qcow2",
 
213
             "is_public": True,
 
214
             "properties": None},
 
215
            create_arguments)
 
216
 
 
217
    def test_prepare_glance_arguments_disk_format(self):
 
218
        # Disk format is based on the image 'ftype' (if defined).
 
219
        source_entry = {"ftype": "root.tar.gz"}
 
220
        create_arguments = self.mirror.prepare_glance_arguments(
 
221
            "foobuntu-X", source_entry, image_md5_hash=None, image_size=None,
 
222
            image_properties=None)
 
223
 
 
224
        self.assertEqual("root-tar", create_arguments["disk_format"])
 
225
 
 
226
    def test_prepare_glance_arguments_size(self):
 
227
        # Size is read from image metadata if defined.
 
228
        source_entry = {"size": 5}
 
229
        create_arguments = self.mirror.prepare_glance_arguments(
 
230
            "foobuntu-X", source_entry, image_md5_hash=None, image_size=None,
 
231
            image_properties=None)
 
232
 
 
233
        self.assertEqual(5, create_arguments["size"])
 
234
 
 
235
    def test_prepare_glance_arguments_checksum(self):
 
236
        # Checksum is based on the source entry 'md5' value, if defined.
 
237
        source_entry = {"md5": "foo123"}
 
238
        create_arguments = self.mirror.prepare_glance_arguments(
 
239
            "foobuntu-X", source_entry, image_md5_hash=None, image_size=None,
 
240
            image_properties=None)
 
241
 
 
242
        self.assertEqual("foo123", create_arguments["checksum"])
 
243
 
 
244
    def test_prepare_glance_arguments_size_and_md5_override(self):
 
245
        # Size and md5 hash are overridden from the passed-in values even if
 
246
        # defined on the source entry.
 
247
        source_entry = {"size": 5, "md5": "foo123"}
 
248
        create_arguments = self.mirror.prepare_glance_arguments(
 
249
            "foobuntu-X", source_entry, image_md5_hash="bar456", image_size=10,
 
250
            image_properties=None)
 
251
 
 
252
        self.assertEqual(10, create_arguments["size"])
 
253
        self.assertEqual("bar456", create_arguments["checksum"])
 
254
 
 
255
    def test_prepare_glance_arguments_size_and_md5_no_override_hash(self):
 
256
        # If only one of image_md5_hash or image_size is passed directly in,
 
257
        # the other value is not overridden either.
 
258
        source_entry = {"size": 5, "md5": "foo123"}
 
259
        create_arguments = self.mirror.prepare_glance_arguments(
 
260
            "foobuntu-X", source_entry, image_md5_hash="bar456",
 
261
            image_size=None, image_properties=None)
 
262
 
 
263
        self.assertEqual(5, create_arguments["size"])
 
264
        self.assertEqual("foo123", create_arguments["checksum"])
 
265
 
 
266
    def test_prepare_glance_arguments_size_and_md5_no_override_size(self):
 
267
        # If only one of image_md5_hash or image_size is passed directly in,
 
268
        # the other value is not overridden either.
 
269
        source_entry = {"size": 5, "md5": "foo123"}
 
270
        create_arguments = self.mirror.prepare_glance_arguments(
 
271
            "foobuntu-X", source_entry, image_md5_hash=None, image_size=10,
 
272
            image_properties=None)
 
273
 
 
274
        self.assertEqual(5, create_arguments["size"])
 
275
        self.assertEqual("foo123", create_arguments["checksum"])
 
276
 
 
277
    def test_download_image(self):
 
278
        # Downloads image from a contentsource.
 
279
        content = "foo bazes the bar"
 
280
        content_source = MemoryContentSource(
 
281
            url="http://image-store/fooubuntu-X-disk1.img", content=content)
 
282
        image_metadata = {"pubname": "foobuntu-X", "size": 5}
 
283
        path, size, md5_hash = self.mirror.download_image(
 
284
            content_source, image_metadata)
 
285
        self.addCleanup(os.unlink, path)
 
286
        self.assertIsNotNone(path)
 
287
        self.assertIsNone(size)
 
288
        self.assertIsNone(md5_hash)
 
289
 
 
290
    def test_download_image_progress_callback(self):
 
291
        # Progress callback is called with image name, size, status and buffer
 
292
        # size after every 10kb of data: 3 times for 25kb of data below.
 
293
        content = "abcdefghij" * int(1024 * 2.5)
 
294
        content_source = MemoryContentSource(
 
295
            url="http://image-store/fooubuntu-X-disk1.img", content=content)
 
296
        image_metadata = {"pubname": "foobuntu-X", "size": len(content)}
 
297
 
 
298
        self.progress_calls = []
 
299
 
 
300
        def log_progress_calls(message):
 
301
            self.progress_calls.append(message)
 
302
 
 
303
        self.addCleanup(
 
304
            setattr, self.mirror, "progress_callback",
 
305
            self.mirror.progress_callback)
 
306
        self.mirror.progress_callback = log_progress_calls
 
307
        path, size, md5_hash = self.mirror.download_image(
 
308
            content_source, image_metadata)
 
309
        self.addCleanup(os.unlink, path)
 
310
 
 
311
        self.assertEqual(
 
312
            [{"name": "foobuntu-X", "size": 25600, "status": "Downloading",
 
313
              "written": 10240}] * 3,
 
314
            self.progress_calls)
 
315
 
 
316
    def test_download_image_error(self):
 
317
        # When there's an error during download, contentsource is still closed
 
318
        # and the error is propagated below.
 
319
        content = "abcdefghij"
 
320
        content_source = MemoryContentSource(
 
321
            url="http://image-store/fooubuntu-X-disk1.img", content=content)
 
322
        image_metadata = {"pubname": "foobuntu-X", "size": len(content)}
 
323
 
 
324
        # MemoryContentSource has an internal file descriptor which indicates
 
325
        # if close() method has been called on it.
 
326
        self.assertFalse(content_source.fd.closed)
 
327
 
 
328
        self.addCleanup(
 
329
            setattr, self.mirror, "progress_callback",
 
330
            self.mirror.progress_callback)
 
331
        self.mirror.progress_callback = lambda message: 1/0
 
332
 
 
333
        self.assertRaises(
 
334
            ZeroDivisionError,
 
335
            self.mirror.download_image, content_source, image_metadata)
 
336
 
 
337
        # We rely on the MemoryContentSource.close() side-effect to ensure
 
338
        # close() method has indeed been called on the passed-in ContentSource.
 
339
        self.assertTrue(content_source.fd.closed)
 
340
 
 
341
    def test_insert_item(self):
 
342
        # Downloads an image from a contentsource, uploads it into Glance,
 
343
        # adapting and munging as needed (it updates the keystone endpoint,
 
344
        # image and owner ids).
 
345
        # This test is basically an integration test to make sure all the
 
346
        # methods used by insert_item() are tied together in one good
 
347
        # fully functioning whole.
 
348
 
 
349
        # This is a real snippet from the simplestreams index entry for
 
350
        # Ubuntu 14.04 amd64 image from cloud-images.ubuntu.com as of
 
351
        # 2016-06-05.
 
352
        source_index = {
 
353
            u'content_id': u'com.ubuntu.cloud:released:download',
 
354
            u'datatype': u'image-downloads',
 
355
            u'format': u'products:1.0',
 
356
            u'license': (u'http://www.canonical.com/'
 
357
                         u'intellectual-property-policy'),
 
358
            u'products': {u'com.ubuntu.cloud:server:14.04:amd64': {
 
359
                u'aliases': u'14.04,default,lts,t,trusty',
 
360
                u'arch': u'amd64',
 
361
                u'os': u'ubuntu',
 
362
                u'release': u'trusty',
 
363
                u'release_codename': u'Trusty Tahr',
 
364
                u'release_title': u'14.04 LTS',
 
365
                u'support_eol': u'2019-04-17',
 
366
                u'supported': True,
 
367
                u'version': u'14.04',
 
368
                u'versions': {u'20160602': {
 
369
                    u'items': {u'disk1.img': {
 
370
                        u'ftype': u'disk1.img',
 
371
                        u'md5': u'e5436cd36ae6cc298f081bf0f6b413f1',
 
372
                        u'path': (
 
373
                            u'server/releases/trusty/release-20160602/'
 
374
                            u'ubuntu-14.04-server-cloudimg-amd64-disk1.img'),
 
375
                        u'sha256': (u'5b982d7d4dd1a03e88ae5f35f02ed44f'
 
376
                                    u'579e2711f3e0f27ea2bff20aef8c8d9e'),
 
377
                        u'size': 259850752}},
 
378
                    u'label': u'release',
 
379
                    u'pubname': u'ubuntu-trusty-14.04-amd64-server-20160602',
 
380
                }}}
 
381
            }
 
382
        }
 
383
 
 
384
        # "Pedigree" is basically a "path" to get to the image data in
 
385
        # simplestreams index, going through "products", their "versions",
 
386
        # and nested "items".
 
387
        pedigree = (
 
388
            u'com.ubuntu.cloud:server:14.04:amd64', u'20160602', u'disk1.img')
 
389
        product = source_index[u'products'][pedigree[0]]
 
390
        image_data = product[u'versions'][pedigree[1]][u'items'][pedigree[2]]
 
391
 
 
392
        content_source = MemoryContentSource(
 
393
            url="http://image-store/fooubuntu-X-disk1.img",
 
394
            content="image-data")
 
395
 
 
396
        # Use a fake GlanceClient to track arguments passed into
 
397
        # GlanceClient.images.create().
 
398
        self.addCleanup(setattr, self.mirror, "gclient", self.mirror.gclient)
 
399
        self.mirror.gclient = FakeGlanceClient()
 
400
 
 
401
        target = {
 
402
            'content_id': 'auto.sync',
 
403
            'datatype': 'image-ids',
 
404
            'format': 'products:1.0',
 
405
        }
 
406
 
 
407
        self.mirror.insert_item(
 
408
            image_data, source_index, target, pedigree, content_source)
 
409
 
 
410
        passed_create_kwargs = self.mirror.gclient.images.create_calls[0]
 
411
 
 
412
        # There is a 'data' argument pointing to an open file descriptor
 
413
        # for the locally downloaded image.
 
414
        self.assertIn("data", passed_create_kwargs)
 
415
        passed_create_kwargs.pop("data")
 
416
 
 
417
        expected_create_kwargs = {
 
418
            'name': ('auto-sync/'
 
419
                     'ubuntu-trusty-14.04-amd64-server-20160602-disk1.img'),
 
420
            'checksum': u'e5436cd36ae6cc298f081bf0f6b413f1',
 
421
            'disk_format': 'qcow2',
 
422
            'container_format': 'bare',
 
423
            'is_public': True,
 
424
            'properties': {
 
425
                'os_distro': u'ubuntu',
 
426
                'item_name': u'disk1.img',
 
427
                'os_version': u'14.04',
 
428
                'architecture': 'x86_64',
 
429
                'version_name': u'20160602',
 
430
                'content_id': 'auto.sync',
 
431
                'product_name': u'com.ubuntu.cloud:server:14.04:amd64',
 
432
                'source_content_id': u'com.ubuntu.cloud:released:download'},
 
433
            'size': '259850752'}
 
434
        self.assertEqual(
 
435
            expected_create_kwargs, passed_create_kwargs)
 
436
 
 
437
        # Almost real resulting data as produced by simplestreams before
 
438
        # insert_item refactoring to allow for finer-grained testing.
 
439
        expected_target_index = {
 
440
            'content_id': 'auto.sync',
 
441
            'datatype': 'image-ids',
 
442
            'format': 'products:1.0',
 
443
            'products': {
 
444
                "com.ubuntu.cloud:server:14.04:amd64": {
 
445
                    "aliases": "14.04,default,lts,t,trusty",
 
446
                    "arch": "amd64",
 
447
                    "label": "release",
 
448
                    "os": "ubuntu",
 
449
                    "owner_id": "bar456",
 
450
                    "pubname": "ubuntu-trusty-14.04-amd64-server-20160602",
 
451
                    "release": "trusty",
 
452
                    "release_codename": "Trusty Tahr",
 
453
                    "release_title": "14.04 LTS",
 
454
                    "support_eol": "2019-04-17",
 
455
                    "supported": "True",
 
456
                    "version": "14.04",
 
457
                    "versions": {"20160602": {"items": {"disk1.img": {
 
458
                        "endpoint": "http://keystone/api/",
 
459
                        "ftype": "disk1.img",
 
460
                        "id": "image-1",
 
461
                        "md5": "e5436cd36ae6cc298f081bf0f6b413f1",
 
462
                        "name": ("auto-sync/ubuntu-trusty-14.04-amd64-"
 
463
                                 "server-20160602-disk1.img"),
 
464
                        "region": "region1",
 
465
                        "sha256": ("5b982d7d4dd1a03e88ae5f35f02ed44f"
 
466
                                   "579e2711f3e0f27ea2bff20aef8c8d9e"),
 
467
                        "size": "259850752"
 
468
                    }}}}
 
469
                }
 
470
            }
 
471
        }
 
472
 
 
473
        # Apply the condensing as done in GlanceMirror.insert_products()
 
474
        # to ensure we compare with the desired resulting simplestreams data.
 
475
        sticky = ['ftype', 'md5', 'sha256', 'size', 'name', 'id', 'endpoint',
 
476
                  'region']
 
477
        simplestreams.util.products_condense(target, sticky)
 
478
 
 
479
        self.assertEqual(expected_target_index, target)