~ubuntu-branches/ubuntu/utopic/maas/utopic-security

« back to all changes in this revision

Viewing changes to src/maasserver/tests/test_bootresources.py

  • Committer: Package Import Robot
  • Author(s): Andres Rodriguez, Jeroen Vermeulen, Andres Rodriguez, Jason Hobbs, Raphaël Badin, Louis Bouchard, Gavin Panella
  • Date: 2014-08-21 19:36:30 UTC
  • mfrom: (1.3.1)
  • Revision ID: package-import@ubuntu.com-20140821193630-kertpu5hd8yyss8h
Tags: 1.7.0~beta7+bzr3266-0ubuntu1
* New Upstream Snapshot, Beta 7 bzr3266

[ Jeroen Vermeulen ]
* debian/extras/99-maas-sudoers
  debian/maas-dhcp.postinst
  debian/rules
  - Add second DHCP server instance for IPv6.
* debian/maas-region-controller-min.install
  debian/maas-region-controller-min.lintian-overrides
  - Install deployment user-data: maas_configure_interfaces.py script.
* debian/maas-cluster-controller.links
  debian/maas-cluster-controller.install
  debian/maas-cluster-controller.postinst
  - Reflect Celery removal changes made in trunk r3067.
  - Don't install celeryconfig_cluster.py any longer. 
  - Don't install maas_local_celeryconfig_cluster.py any longer.
  - Don't symlink maas_local_celeryconfig_cluster.py from /etc to /usr.
  - Don't insert UUID into maas_local_celeryconfig_cluster.py.

[ Andres Rodriguez ]
* debian/maas-region-controller-min.postrm: Cleanup lefover files.
* debian/maas-dhcp.postrm: Clean leftover configs.
* Provide new maas-proxy package that replaces the usage of
  squid-deb-proxy:
  - debian/control: New maas-proxy package that replaces the usage
    of squid-deb-proxy; Drop depends on squid-deb-proxy.
  - Add upstrart job.
  - Ensure squid3 is stopped as maas-proxy uses a caching proxy.
* Remove Celery references to cluster controller:
  - Rename upstart job from maas-pserv to maas-cluster; rename
    maas-cluster-celery to maas-cluster-register. Ensure services
    are stopped on upgrade.
  - debian/maintscript: Cleanup config files.
  - Remove all references to the MAAS celery daemon and config
    files as we don't use it like that anymore
* Move some entries in debian/maintscript to
  debian/maas-cluster-controller.maintscript
* Remove usage of txlongpoll and rabbitmq-server. Handle upgrades
  to ensure these are removed correctly.

[ Jason Hobbs ]
* debian/maas-region-controller-min.install: Install
  maas-generate-winrm-cert script.

[ Raphaël Badin ]
* debian/extras/maas-region-admin: Bypass django-admin as it prints
  spurious messages to stdout (LP: #1365130).

[Louis Bouchard]
* debian/maas-cluster-controller.postinst:
  - Exclude /var/log/maas/rsyslog when changing ownership
    (LP: #1346703)

[Gavin Panella]
* debian/maas-cluster-controller.maas-clusterd.upstart:
  - Don't start-up the cluster controller unless a shared-secret has
    been installed.
* debian/maas-cluster-controller.maas-cluster-register.upstart: Drop.

Show diffs side-by-side

added added

removed removed

Lines of Context:
16
16
 
17
17
import httplib
18
18
import json
 
19
from os import environ
19
20
from random import randint
20
21
from StringIO import StringIO
21
22
 
22
23
from django.core.urlresolvers import reverse
23
 
from django.db import transaction
 
24
from django.db import (
 
25
    connections,
 
26
    transaction,
 
27
    )
24
28
from django.http import StreamingHttpResponse
25
29
from django.test.client import Client
 
30
from maasserver import bootresources
26
31
from maasserver.bootresources import (
27
 
    ensure_boot_source_definition,
 
32
    BootResourceStore,
 
33
    download_all_boot_resources,
 
34
    download_boot_resources,
28
35
    get_simplestream_endpoint,
29
36
    )
30
37
from maasserver.enum import (
32
39
    BOOT_RESOURCE_TYPE,
33
40
    )
34
41
from maasserver.models import (
35
 
    BootSource,
36
 
    BootSourceSelection,
 
42
    BootResource,
 
43
    BootResourceFile,
 
44
    BootResourceSet,
 
45
    Config,
 
46
    LargeFile,
37
47
    )
38
48
from maasserver.testing.factory import factory
 
49
from maasserver.testing.orm import reload_object
39
50
from maasserver.testing.testcase import MAASServerTestCase
40
51
from maasserver.utils import absolute_reverse
 
52
from maasserver.utils.orm import get_one
 
53
from maastesting.djangotestcase import TransactionTestCase
 
54
from maastesting.matchers import (
 
55
    MockCalledOnceWith,
 
56
    MockNotCalled,
 
57
    )
41
58
from maastesting.testcase import MAASTestCase
42
 
from testtools.matchers import (
43
 
    ContainsAll,
44
 
    HasLength,
 
59
from mock import (
 
60
    ANY,
 
61
    MagicMock,
 
62
    Mock,
 
63
    sentinel,
45
64
    )
 
65
from provisioningserver.import_images.product_mapping import ProductMapping
 
66
from testtools.matchers import ContainsAll
46
67
 
47
68
 
48
69
def make_boot_resource_file_with_stream():
68
89
            endpoint['url'])
69
90
        self.assertEqual([], endpoint['selections'])
70
91
 
71
 
    def test_ensure_boot_source_definition_creates_default_source(self):
72
 
        ensure_boot_source_definition()
73
 
        sources = BootSource.objects.all()
74
 
        self.assertThat(sources, HasLength(1))
75
 
        [source] = sources
76
 
        self.assertAttributes(
77
 
            source,
78
 
            {
79
 
                'url': 'http://maas.ubuntu.com/images/ephemeral-v2/releases/',
80
 
                'keyring_filename': (
81
 
                    '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'),
82
 
            })
83
 
        selections = BootSourceSelection.objects.filter(boot_source=source)
84
 
        by_release = {
85
 
            selection.release: selection
86
 
            for selection in selections
87
 
            }
88
 
        self.assertItemsEqual(['trusty'], by_release.keys())
89
 
        self.assertAttributes(
90
 
            by_release['trusty'],
91
 
            {
92
 
                'release': 'trusty',
93
 
                'arches': ['amd64'],
94
 
                'subarches': ['*'],
95
 
                'labels': ['release'],
96
 
            })
97
 
 
98
 
    def test_ensure_boot_source_definition_skips_if_already_present(self):
99
 
        sources = [
100
 
            factory.make_boot_source()
101
 
            for _ in range(3)
102
 
            ]
103
 
        ensure_boot_source_definition()
104
 
        self.assertItemsEqual(sources, BootSource.objects.all())
105
 
 
106
92
 
107
93
class TestSimpleStreamsHandler(MAASServerTestCase):
108
94
    """Tests for `maasserver.bootresources.SimpleStreamsHandler`."""
204
190
            output['index']['maas:v2:download']['products'])
205
191
 
206
192
    def test_streams_product_index_empty_with_incomplete_resource(self):
207
 
        resource = factory.make_boot_resource()
208
 
        factory.make_boot_resource_set(resource)
 
193
        resource = factory.make_BootResource()
 
194
        factory.make_BootResourceSet(resource)
209
195
        response = self.get_stream_client('index.json')
210
196
        output = json.loads(response.content)
211
197
        self.assertEqual(
246
232
            output['products'])
247
233
 
248
234
    def test_streams_product_download_empty_with_incomplete_resource(self):
249
 
        resource = factory.make_boot_resource()
250
 
        factory.make_boot_resource_set(resource)
 
235
        resource = factory.make_BootResource()
 
236
        factory.make_BootResourceSet(resource)
251
237
        response = self.get_stream_client('maas:v2:download.json')
252
238
        output = json.loads(response.content)
253
239
        self.assertEqual(
297
283
    def test_streams_product_download_product_uses_latest_complete_label(self):
298
284
        product, resource = self.make_usable_product_boot_resource()
299
285
        # Incomplete resource_set
300
 
        factory.make_boot_resource_set(resource)
301
 
        newest_set = factory.make_boot_resource_set(resource)
 
286
        factory.make_BootResourceSet(resource)
 
287
        newest_set = factory.make_BootResourceSet(resource)
302
288
        factory.make_boot_resource_file_with_content(newest_set)
303
289
        response = self.get_stream_client('maas:v2:download.json')
304
290
        output = json.loads(response.content)
306
292
        self.assertEqual(newest_set.label, output_product['label'])
307
293
 
308
294
    def test_streams_product_download_product_contains_multiple_versions(self):
309
 
        resource = factory.make_boot_resource()
 
295
        resource = factory.make_BootResource()
310
296
        resource_sets = [
311
 
            factory.make_boot_resource_set(resource)
 
297
            factory.make_BootResourceSet(resource)
312
298
            for _ in range(3)
313
299
            ]
314
300
        versions = []
414
400
        version = resource_set.version
415
401
        resource_file = resource_set.files.order_by('?')[0]
416
402
        filename = resource_file.filename
417
 
        with resource_file.largefile.content.open('rb') as stream:
418
 
            content = stream.read()
419
403
        response = self.get_file_client(
420
404
            os, arch, subarch, series, version, filename)
421
405
        self.assertIsInstance(response, StreamingHttpResponse)
422
 
        self.assertEqual(content, b''.join(response.streaming_content))
423
 
 
424
 
 
425
 
class TestTransactionWrapper(MAASTestCase):
426
 
    """Tests the use of StreamingHttpResponse(TransactionWrapper(stream)).
 
406
 
 
407
 
 
408
class TestConnectionWrapper(TransactionTestCase):
 
409
    """Tests the use of StreamingHttpResponse(ConnectionWrapper(stream)).
427
410
 
428
411
    We do not run this inside of `MAASServerTestCase` as that wraps a
429
 
    transaction around each test. This removes that behavior so we can
430
 
    test that the transaction is remaining open for all of the content.
 
412
    transaction around each test. Since a new connection is created to return
 
413
    the actual content, the transaction to create the data needs be committed.
431
414
    """
432
415
 
433
 
    def test_download(self):
434
 
        # Do the setup inside of a transaction, as we are running in a test
435
 
        # that doesn't enable transactions per test.
 
416
    def make_file_for_client(self):
 
417
        # Set up the database information inside of a transaction. This is
 
418
        # done so the information is committed. As the new connection needs
 
419
        # to be able to access the data.
436
420
        with transaction.atomic():
437
421
            os = factory.make_name('os')
438
422
            series = factory.make_name('series')
448
432
            filename = filetype
449
433
            size = randint(1024, 2048)
450
434
            content = factory.make_string(size=size)
451
 
            resource = factory.make_boot_resource(
 
435
            resource = factory.make_BootResource(
452
436
                rtype=BOOT_RESOURCE_TYPE.SYNCED, name=name,
453
437
                architecture=architecture)
454
 
            resource_set = factory.make_boot_resource_set(
 
438
            resource_set = factory.make_BootResourceSet(
455
439
                resource, version=version)
456
 
            largefile = factory.make_large_file(content=content, size=size)
457
 
            factory.make_boot_resource_file(
 
440
            largefile = factory.make_LargeFile(content=content, size=size)
 
441
            factory.make_BootResourceFile(
458
442
                resource_set, largefile, filename=filename, filetype=filetype)
459
 
 
460
 
        # Outside of the transaction, we run the actual test. The client will
461
 
        # run inside of its own transaction, but once the streaming response
462
 
        # is returned that transaction will be closed.
463
 
        client = Client()
464
 
        response = client.get(
465
 
            reverse(
466
 
                'simplestreams_file_handler', kwargs={
467
 
                    'os': os,
468
 
                    'arch': arch,
469
 
                    'subarch': subarch,
470
 
                    'series': series,
471
 
                    'version': version,
472
 
                    'filename': filename,
473
 
                    }))
474
 
 
475
 
        # If TransactionWrapper does not work, then a ProgramError will be
476
 
        # thrown. If it works then content will match.
477
 
        self.assertEqual(content, b''.join(response.streaming_content))
478
 
        self.assertTrue(largefile.content.closed)
 
443
        return content, reverse(
 
444
            'simplestreams_file_handler', kwargs={
 
445
                'os': os,
 
446
                'arch': arch,
 
447
                'subarch': subarch,
 
448
                'series': series,
 
449
                'version': version,
 
450
                'filename': filename,
 
451
                })
 
452
 
 
453
    def read_response(self, response):
 
454
        """Read the streaming_content from the response.
 
455
 
 
456
        :rtype: bytes
 
457
        """
 
458
        return b''.join(response.streaming_content)
 
459
 
 
460
    def test_download_calls__get_new_connection(self):
 
461
        content, url = self.make_file_for_client()
 
462
        mock_get_new_connection = self.patch(
 
463
            bootresources.ConnectionWrapper, '_get_new_connection')
 
464
 
 
465
        client = Client()
 
466
        response = client.get(url)
 
467
        self.read_response(response)
 
468
        self.assertThat(mock_get_new_connection, MockCalledOnceWith())
 
469
 
 
470
    def test_download_connection_is_not_same_as_django_connections(self):
 
471
        content, url = self.make_file_for_client()
 
472
 
 
473
        class AssertConnectionWrapper(bootresources.ConnectionWrapper):
 
474
 
 
475
            def _set_up(self):
 
476
                super(AssertConnectionWrapper, self)._set_up()
 
477
                # Capture the created connection
 
478
                AssertConnectionWrapper.connection = self._connection
 
479
 
 
480
            def close(self):
 
481
                # Close the stream, but we don't want to close the
 
482
                # connection as the test is testing that the connection is
 
483
                # not the same as the connection django is using for other
 
484
                # webrequests.
 
485
                if self._stream is not None:
 
486
                    self._stream.close()
 
487
                    self._stream = None
 
488
                self._connection = None
 
489
 
 
490
        self.patch(
 
491
            bootresources, 'ConnectionWrapper', AssertConnectionWrapper)
 
492
 
 
493
        client = Client()
 
494
        response = client.get(url)
 
495
        self.read_response(response)
 
496
 
 
497
        # Add cleanup to close the connection, since this was removed from
 
498
        # AssertConnectionWrapper.close method.
 
499
        def close():
 
500
            conn = AssertConnectionWrapper.connection
 
501
            conn.commit()
 
502
            conn.leave_transaction_management()
 
503
            conn.close()
 
504
        self.addCleanup(close)
 
505
 
 
506
        # The connection that is used by the wrapper cannot be the same as the
 
507
        # connection be using for all other webrequests. Without this
 
508
        # seperate the transactional middleware will fail to initialize,
 
509
        # because the the connection will already be in a transaction.
 
510
        #
 
511
        # Note: cannot test if DatabaseWrapper != DatabaseWrapper, as it will
 
512
        # report true, because the __eq__ operator only checks if the aliases
 
513
        # are the same. This is checking the underlying connection is
 
514
        # different, which is the important part.
 
515
        self.assertNotEqual(
 
516
            connections["default"].connection,
 
517
            AssertConnectionWrapper.connection.connection)
 
518
 
 
519
 
 
520
def make_product():
 
521
    """Make product dictionary that is just like the one provided
 
522
    from simplsetreams."""
 
523
    subarch = factory.make_name('subarch')
 
524
    subarches = [factory.make_name('subarch') for _ in range(3)]
 
525
    subarches.insert(0, subarch)
 
526
    subarches = ','.join(subarches)
 
527
    product = {
 
528
        'os': factory.make_name('os'),
 
529
        'arch': factory.make_name('arch'),
 
530
        'subarch': subarch,
 
531
        'release': factory.make_name('release'),
 
532
        'kflavor': factory.make_name('kflavor'),
 
533
        'subarches': subarches,
 
534
        'version_name': factory.make_name('version'),
 
535
        'label': factory.make_name('label'),
 
536
        'ftype': factory.pick_enum(BOOT_RESOURCE_FILE_TYPE),
 
537
        'kpackage': factory.make_name('kpackage'),
 
538
        'di_version': factory.make_name('di_version'),
 
539
        }
 
540
    name = '%s/%s' % (product['os'], product['release'])
 
541
    architecture = '%s/%s' % (product['arch'], product['subarch'])
 
542
    return name, architecture, product
 
543
 
 
544
 
 
545
def make_boot_resource_group(
 
546
        rtype=None, name=None, architecture=None,
 
547
        version=None, filename=None, filetype=None):
 
548
    """Make boot resource that contains one set and one file."""
 
549
    resource = factory.make_BootResource(
 
550
        rtype=rtype, name=name, architecture=architecture)
 
551
    resource_set = factory.make_BootResourceSet(resource, version=version)
 
552
    rfile = factory.make_boot_resource_file_with_content(
 
553
        resource_set, filename=filename, filetype=filetype)
 
554
    return resource, resource_set, rfile
 
555
 
 
556
 
 
557
def make_boot_resource_group_from_product(product):
 
558
    """Make boot resource that contains one set and one file, using the
 
559
    information from the given product.
 
560
 
 
561
    The product dictionary is also updated to include the sha256 and size
 
562
    for the created largefile. The calling function should use the returned
 
563
    product in place of the passed product.
 
564
    """
 
565
    name = '%s/%s' % (product['os'], product['release'])
 
566
    architecture = '%s/%s' % (product['arch'], product['subarch'])
 
567
    resource = factory.make_BootResource(
 
568
        rtype=BOOT_RESOURCE_TYPE.SYNCED, name=name,
 
569
        architecture=architecture)
 
570
    resource_set = factory.make_BootResourceSet(
 
571
        resource, version=product['version_name'])
 
572
    rfile = factory.make_boot_resource_file_with_content(
 
573
        resource_set, filename=product['ftype'],
 
574
        filetype=product['ftype'])
 
575
    product['sha256'] = rfile.largefile.sha256
 
576
    product['size'] = rfile.largefile.total_size
 
577
    return product, resource
 
578
 
 
579
 
 
580
class TestBootResourceStore(MAASServerTestCase):
 
581
 
 
582
    def make_boot_resources(self):
 
583
        resources = [
 
584
            factory.make_BootResource(rtype=BOOT_RESOURCE_TYPE.SYNCED)
 
585
            for _ in range(3)
 
586
            ]
 
587
        resource_names = []
 
588
        for resource in resources:
 
589
            os, series = resource.name.split('/')
 
590
            arch, subarch = resource.split_arch()
 
591
            name = '%s/%s/%s/%s' % (os, arch, subarch, series)
 
592
            resource_names.append(name)
 
593
        return resources, resource_names
 
594
 
 
595
    def test_init_initializes_variables(self):
 
596
        _, resource_names = self.make_boot_resources()
 
597
        store = BootResourceStore()
 
598
        self.assertItemsEqual(resource_names, store._resources_to_delete)
 
599
        self.assertEqual({}, store._content_to_finalize)
 
600
 
 
601
    def test_prevent_resource_deletion_removes_resource(self):
 
602
        resources, resource_names = self.make_boot_resources()
 
603
        store = BootResourceStore()
 
604
        resource = resources.pop()
 
605
        resource_names.pop()
 
606
        store.prevent_resource_deletion(resource)
 
607
        self.assertItemsEqual(resource_names, store._resources_to_delete)
 
608
 
 
609
    def test_prevent_resource_deletion_doesnt_remove_unknown_resource(self):
 
610
        resources, resource_names = self.make_boot_resources()
 
611
        store = BootResourceStore()
 
612
        resource = factory.make_BootResource(rtype=BOOT_RESOURCE_TYPE.SYNCED)
 
613
        store.prevent_resource_deletion(resource)
 
614
        self.assertItemsEqual(resource_names, store._resources_to_delete)
 
615
 
 
616
    def test_save_content_later_adds_to__content_to_finalize_var(self):
 
617
        _, _, rfile = make_boot_resource_group()
 
618
        store = BootResourceStore()
 
619
        store.save_content_later(rfile, sentinel.reader)
 
620
        self.assertEqual(
 
621
            {rfile.id: sentinel.reader},
 
622
            store._content_to_finalize)
 
623
 
 
624
    def test_get_or_create_boot_resource_creates_resource(self):
 
625
        name, architecture, product = make_product()
 
626
        store = BootResourceStore()
 
627
        resource = store.get_or_create_boot_resource(product)
 
628
        self.assertEqual(BOOT_RESOURCE_TYPE.SYNCED, resource.rtype)
 
629
        self.assertEqual(name, resource.name)
 
630
        self.assertEqual(architecture, resource.architecture)
 
631
        self.assertEqual(product['kflavor'], resource.extra['kflavor'])
 
632
        self.assertEqual(product['subarches'], resource.extra['subarches'])
 
633
 
 
634
    def test_get_or_create_boot_resource_gets_resource(self):
 
635
        name, architecture, product = make_product()
 
636
        expected = factory.make_BootResource(
 
637
            rtype=BOOT_RESOURCE_TYPE.SYNCED, name=name,
 
638
            architecture=architecture)
 
639
        store = BootResourceStore()
 
640
        resource = store.get_or_create_boot_resource(product)
 
641
        self.assertEqual(expected, resource)
 
642
        self.assertEqual(product['kflavor'], resource.extra['kflavor'])
 
643
        self.assertEqual(product['subarches'], resource.extra['subarches'])
 
644
 
 
645
    def test_get_or_create_boot_resource_calls_prevent_resource_deletion(self):
 
646
        name, architecture, product = make_product()
 
647
        resource = factory.make_BootResource(
 
648
            rtype=BOOT_RESOURCE_TYPE.SYNCED,
 
649
            name=name, architecture=architecture)
 
650
        store = BootResourceStore()
 
651
        mock_prevent = self.patch(store, 'prevent_resource_deletion')
 
652
        store.get_or_create_boot_resource(product)
 
653
        self.assertThat(
 
654
            mock_prevent, MockCalledOnceWith(resource))
 
655
 
 
656
    def test_get_or_create_boot_resource_converts_generated_into_synced(self):
 
657
        name, architecture, product = make_product()
 
658
        resource = factory.make_BootResource(
 
659
            rtype=BOOT_RESOURCE_TYPE.GENERATED,
 
660
            name=name, architecture=architecture)
 
661
        store = BootResourceStore()
 
662
        mock_prevent = self.patch(store, 'prevent_resource_deletion')
 
663
        store.get_or_create_boot_resource(product)
 
664
        self.assertEqual(
 
665
            BOOT_RESOURCE_TYPE.SYNCED,
 
666
            reload_object(resource).rtype)
 
667
        self.assertThat(
 
668
            mock_prevent, MockNotCalled())
 
669
 
 
670
    def test_get_or_create_boot_resource_set_creates_resource_set(self):
 
671
        name, architecture, product = make_product()
 
672
        product, resource = make_boot_resource_group_from_product(product)
 
673
        resource.sets.all().delete()
 
674
        store = BootResourceStore()
 
675
        resource_set = store.get_or_create_boot_resource_set(resource, product)
 
676
        self.assertEqual(product['version_name'], resource_set.version)
 
677
        self.assertEqual(product['label'], resource_set.label)
 
678
 
 
679
    def test_get_or_create_boot_resource_set_gets_resource_set(self):
 
680
        name, architecture, product = make_product()
 
681
        product, resource = make_boot_resource_group_from_product(product)
 
682
        expected = resource.sets.first()
 
683
        store = BootResourceStore()
 
684
        resource_set = store.get_or_create_boot_resource_set(resource, product)
 
685
        self.assertEqual(expected, resource_set)
 
686
        self.assertEqual(product['label'], resource_set.label)
 
687
 
 
688
    def test_get_or_create_boot_resource_file_creates_resource_file(self):
 
689
        name, architecture, product = make_product()
 
690
        product, resource = make_boot_resource_group_from_product(product)
 
691
        resource_set = resource.sets.first()
 
692
        resource_set.files.all().delete()
 
693
        store = BootResourceStore()
 
694
        rfile = store.get_or_create_boot_resource_file(resource_set, product)
 
695
        self.assertEqual(product['ftype'], rfile.filename)
 
696
        self.assertEqual(product['ftype'], rfile.filetype)
 
697
        self.assertEqual(product['kpackage'], rfile.extra['kpackage'])
 
698
        self.assertEqual(product['di_version'], rfile.extra['di_version'])
 
699
 
 
700
    def test_get_or_create_boot_resource_file_gets_resource_file(self):
 
701
        name, architecture, product = make_product()
 
702
        product, resource = make_boot_resource_group_from_product(product)
 
703
        resource_set = resource.sets.first()
 
704
        expected = resource_set.files.first()
 
705
        store = BootResourceStore()
 
706
        rfile = store.get_or_create_boot_resource_file(resource_set, product)
 
707
        self.assertEqual(expected, rfile)
 
708
        self.assertEqual(product['ftype'], rfile.filetype)
 
709
        self.assertEqual(product['kpackage'], rfile.extra['kpackage'])
 
710
        self.assertEqual(product['di_version'], rfile.extra['di_version'])
 
711
 
 
712
    def test_get_resource_file_log_identifier_returns_valid_ident(self):
 
713
        os = factory.make_name('os')
 
714
        series = factory.make_name('series')
 
715
        arch = factory.make_name('arch')
 
716
        subarch = factory.make_name('subarch')
 
717
        version = factory.make_name('version')
 
718
        filename = factory.make_name('filename')
 
719
        name = '%s/%s' % (os, series)
 
720
        architecture = '%s/%s' % (arch, subarch)
 
721
        resource = factory.make_BootResource(
 
722
            rtype=BOOT_RESOURCE_TYPE.SYNCED, name=name,
 
723
            architecture=architecture)
 
724
        resource_set = factory.make_BootResourceSet(
 
725
            resource, version=version)
 
726
        rfile = factory.make_boot_resource_file_with_content(
 
727
            resource_set, filename=filename)
 
728
        store = BootResourceStore()
 
729
        self.assertEqual(
 
730
            '%s/%s/%s/%s/%s/%s' % (
 
731
                os, arch, subarch, series, version, filename),
 
732
            store.get_resource_file_log_identifier(rfile))
 
733
        self.assertEqual(
 
734
            '%s/%s/%s/%s/%s/%s' % (
 
735
                os, arch, subarch, series, version, filename),
 
736
            store.get_resource_file_log_identifier(
 
737
                rfile, resource_set, resource))
 
738
 
 
739
    def test_write_content_saves_data(self):
 
740
        rfile, reader, content = make_boot_resource_file_with_stream()
 
741
        store = BootResourceStore()
 
742
        store.write_content(rfile, reader)
 
743
        self.assertTrue(BootResourceFile.objects.filter(id=rfile.id).exists())
 
744
        with rfile.largefile.content.open('rb') as stream:
 
745
            written_data = stream.read()
 
746
        self.assertEqual(content, written_data)
 
747
 
 
748
    def test_write_content_deletes_file_on_bad_checksum(self):
 
749
        rfile, _, _ = make_boot_resource_file_with_stream()
 
750
        reader = StringIO(factory.make_string())
 
751
        store = BootResourceStore()
 
752
        store.write_content(rfile, reader)
 
753
        self.assertFalse(BootResourceFile.objects.filter(id=rfile.id).exists())
 
754
 
 
755
    def test_finalize_calls_methods(self):
 
756
        store = BootResourceStore()
 
757
        mock_resource_cleaner = self.patch(store, 'resource_cleaner')
 
758
        mock_perform_write = self.patch(store, 'perform_write')
 
759
        mock_resource_set_cleaner = self.patch(store, 'resource_set_cleaner')
 
760
        store.finalize()
 
761
        self.assertTrue(mock_resource_cleaner, MockCalledOnceWith())
 
762
        self.assertTrue(mock_perform_write, MockCalledOnceWith())
 
763
        self.assertTrue(mock_resource_set_cleaner, MockCalledOnceWith())
 
764
 
 
765
 
 
766
class TestBootResourceTransactional(TransactionTestCase):
 
767
    """Test methods on `BootResourceStore` that manage their own transactions.
 
768
 
 
769
    This is done using TransactionTestCase so the database is flushed after
 
770
    each test run.
 
771
    """
 
772
 
 
773
    def test_insert_does_nothing_if_file_already_exists(self):
 
774
        name, architecture, product = make_product()
 
775
        with transaction.atomic():
 
776
            product, resource = make_boot_resource_group_from_product(product)
 
777
            rfile = resource.sets.first().files.first()
 
778
        largefile = rfile.largefile
 
779
        store = BootResourceStore()
 
780
        mock_save_later = self.patch(store, 'save_content_later')
 
781
        store.insert(product, sentinel.reader)
 
782
        self.assertEqual(largefile, reload_object(rfile).largefile)
 
783
        self.assertThat(mock_save_later, MockNotCalled())
 
784
 
 
785
    def test_insert_uses_already_existing_largefile(self):
 
786
        name, architecture, product = make_product()
 
787
        with transaction.atomic():
 
788
            product, resource = make_boot_resource_group_from_product(product)
 
789
            resource_set = resource.sets.first()
 
790
            resource_set.files.all().delete()
 
791
            largefile = factory.make_LargeFile()
 
792
        product['sha256'] = largefile.sha256
 
793
        product['size'] = largefile.total_size
 
794
        store = BootResourceStore()
 
795
        mock_save_later = self.patch(store, 'save_content_later')
 
796
        store.insert(product, sentinel.reader)
 
797
        self.assertEqual(
 
798
            largefile,
 
799
            get_one(reload_object(resource_set).files.all()).largefile)
 
800
        self.assertThat(mock_save_later, MockNotCalled())
 
801
 
 
802
    def test_insert_deletes_mismatch_largefile(self):
 
803
        name, architecture, product = make_product()
 
804
        with transaction.atomic():
 
805
            product, resource = make_boot_resource_group_from_product(product)
 
806
            rfile = resource.sets.first().files.first()
 
807
            delete_largefile = rfile.largefile
 
808
            largefile = factory.make_LargeFile()
 
809
        product['sha256'] = largefile.sha256
 
810
        product['size'] = largefile.total_size
 
811
        store = BootResourceStore()
 
812
        mock_save_later = self.patch(store, 'save_content_later')
 
813
        store.insert(product, sentinel.reader)
 
814
        self.assertFalse(
 
815
            LargeFile.objects.filter(id=delete_largefile.id).exists())
 
816
        self.assertEqual(largefile, reload_object(rfile).largefile)
 
817
        self.assertThat(mock_save_later, MockNotCalled())
 
818
 
 
819
    def test_insert_deletes_mismatch_largefile_keeps_other_resource_file(self):
 
820
        name, architecture, product = make_product()
 
821
        with transaction.atomic():
 
822
            resource = factory.make_BootResource(
 
823
                rtype=BOOT_RESOURCE_TYPE.SYNCED, name=name,
 
824
                architecture=architecture)
 
825
            resource_set = factory.make_BootResourceSet(
 
826
                resource, version=product['version_name'])
 
827
            other_type = factory.pick_enum(
 
828
                BOOT_RESOURCE_FILE_TYPE, but_not=product['ftype'])
 
829
            other_file = factory.make_boot_resource_file_with_content(
 
830
                resource_set, filename=other_type, filetype=other_type)
 
831
            rfile = factory.make_BootResourceFile(
 
832
                resource_set, other_file.largefile,
 
833
                filename=product['ftype'], filetype=product['ftype'])
 
834
            largefile = factory.make_LargeFile()
 
835
        product['sha256'] = largefile.sha256
 
836
        product['size'] = largefile.total_size
 
837
        store = BootResourceStore()
 
838
        mock_save_later = self.patch(store, 'save_content_later')
 
839
        store.insert(product, sentinel.reader)
 
840
        self.assertEqual(largefile, reload_object(rfile).largefile)
 
841
        self.assertTrue(
 
842
            LargeFile.objects.filter(id=other_file.largefile.id).exists())
 
843
        self.assertTrue(
 
844
            BootResourceFile.objects.filter(id=other_file.id).exists())
 
845
        self.assertEqual(
 
846
            other_file.largefile, reload_object(other_file).largefile)
 
847
        self.assertThat(mock_save_later, MockNotCalled())
 
848
 
 
849
    def test_insert_creates_new_largefile(self):
 
850
        name, architecture, product = make_product()
 
851
        with transaction.atomic():
 
852
            resource = factory.make_BootResource(
 
853
                rtype=BOOT_RESOURCE_TYPE.SYNCED, name=name,
 
854
                architecture=architecture)
 
855
            resource_set = factory.make_BootResourceSet(
 
856
                resource, version=product['version_name'])
 
857
        product['sha256'] = factory.make_string(size=64)
 
858
        product['size'] = randint(1024, 2048)
 
859
        store = BootResourceStore()
 
860
        mock_save_later = self.patch(store, 'save_content_later')
 
861
        store.insert(product, sentinel.reader)
 
862
        rfile = get_one(reload_object(resource_set).files.all())
 
863
        self.assertEqual(product['sha256'], rfile.largefile.sha256)
 
864
        self.assertEqual(product['size'], rfile.largefile.total_size)
 
865
        self.assertThat(
 
866
            mock_save_later,
 
867
            MockCalledOnceWith(rfile, sentinel.reader))
 
868
 
 
869
    def test_resource_cleaner_removes_old_boot_resources(self):
 
870
        with transaction.atomic():
 
871
            resources = [
 
872
                factory.make_BootResource(rtype=BOOT_RESOURCE_TYPE.SYNCED)
 
873
                for _ in range(3)
 
874
                ]
 
875
        store = BootResourceStore()
 
876
        store.resource_cleaner()
 
877
        for resource in resources:
 
878
            os, series = resource.name.split('/')
 
879
            arch, subarch = resource.split_arch()
 
880
            self.assertFalse(
 
881
                BootResource.objects.has_synced_resource(
 
882
                    os, arch, subarch, series))
 
883
 
 
884
    def test_resource_set_cleaner_removes_incomplete_set(self):
 
885
        with transaction.atomic():
 
886
            resource = factory.make_usable_boot_resource(
 
887
                rtype=BOOT_RESOURCE_TYPE.SYNCED)
 
888
            incomplete_set = factory.make_BootResourceSet(resource)
 
889
        store = BootResourceStore()
 
890
        store.resource_set_cleaner()
 
891
        self.assertFalse(
 
892
            BootResourceSet.objects.filter(id=incomplete_set.id).exists())
 
893
 
 
894
    def test_resource_set_cleaner_keeps_only_newest_completed_set(self):
 
895
        with transaction.atomic():
 
896
            resource = factory.make_BootResource(
 
897
                rtype=BOOT_RESOURCE_TYPE.SYNCED)
 
898
            old_complete_sets = []
 
899
            for _ in range(3):
 
900
                resource_set = factory.make_BootResourceSet(resource)
 
901
                factory.make_boot_resource_file_with_content(resource_set)
 
902
                old_complete_sets.append(resource_set)
 
903
            newest_set = factory.make_BootResourceSet(resource)
 
904
            factory.make_boot_resource_file_with_content(newest_set)
 
905
        store = BootResourceStore()
 
906
        store.resource_set_cleaner()
 
907
        self.assertItemsEqual([newest_set], resource.sets.all())
 
908
        for resource_set in old_complete_sets:
 
909
            self.assertFalse(
 
910
                BootResourceSet.objects.filter(id=resource_set.id).exists())
 
911
 
 
912
    def test_resource_set_cleaner_removes_resources_with_empty_sets(self):
 
913
        with transaction.atomic():
 
914
            resource = factory.make_BootResource(
 
915
                rtype=BOOT_RESOURCE_TYPE.SYNCED)
 
916
        store = BootResourceStore()
 
917
        store.resource_set_cleaner()
 
918
        self.assertFalse(
 
919
            BootResource.objects.filter(id=resource.id).exists())
 
920
 
 
921
    def test_perform_writes_writes_all_content(self):
 
922
        with transaction.atomic():
 
923
            files = [make_boot_resource_file_with_stream() for _ in range(3)]
 
924
            store = BootResourceStore()
 
925
            for rfile, reader, content in files:
 
926
                store.save_content_later(rfile, reader)
 
927
        store.perform_write()
 
928
        with transaction.atomic():
 
929
            for rfile, reader, content in files:
 
930
                self.assertTrue(
 
931
                    BootResourceFile.objects.filter(id=rfile.id).exists())
 
932
                with rfile.largefile.content.open('rb') as stream:
 
933
                    written_data = stream.read()
 
934
                self.assertEqual(content, written_data)
 
935
 
 
936
 
 
937
class TestImportImages(MAASTestCase):
 
938
 
 
939
    def patch_and_capture_env_for_download_all_boot_resources(self):
 
940
        class CaptureEnv:
 
941
            """Fake function; records a copy of the environment."""
 
942
 
 
943
            def __call__(self, *args, **kwargs):
 
944
                self.args = args
 
945
                self.env = environ.copy()
 
946
 
 
947
        capture = self.patch(
 
948
            bootresources, 'download_all_boot_resources', CaptureEnv())
 
949
        return capture
 
950
 
 
951
    def test_download_boot_resources_syncs_repo(self):
 
952
        fake_sync = self.patch(bootresources.BootResourceRepoWriter, 'sync')
 
953
        store = BootResourceStore()
 
954
        source_url = factory.make_url()
 
955
        download_boot_resources(
 
956
            source_url, store, None, None)
 
957
        self.assertEqual(1, len(fake_sync.mock_calls))
 
958
 
 
959
    def test_download_all_boot_resources_calls_download_boot_resources(self):
 
960
        source = {
 
961
            'url': factory.make_url(),
 
962
            'keyring': self.make_file("keyring"),
 
963
            }
 
964
        product_mapping = ProductMapping()
 
965
        store = BootResourceStore()
 
966
        fake_download = self.patch(bootresources, 'download_boot_resources')
 
967
        download_all_boot_resources(
 
968
            sources=[source], product_mapping=product_mapping, store=store)
 
969
        self.assertThat(
 
970
            fake_download,
 
971
            MockCalledOnceWith(
 
972
                source['url'], store, product_mapping,
 
973
                keyring_file=source['keyring']))
 
974
 
 
975
    def test_download_all_boot_resources_calls_finalize_on_store(self):
 
976
        product_mapping = ProductMapping()
 
977
        store = BootResourceStore()
 
978
        fake_finalize = self.patch(store, 'finalize')
 
979
        download_all_boot_resources(
 
980
            sources=[], product_mapping=product_mapping, store=store)
 
981
        self.assertThat(
 
982
            fake_finalize,
 
983
            MockCalledOnceWith())
 
984
 
 
985
    def test_has_synced_resources_returns_true(self):
 
986
        factory.make_BootResource(rtype=BOOT_RESOURCE_TYPE.SYNCED)
 
987
        self.assertTrue(bootresources.has_synced_resources())
 
988
 
 
989
    def test_has_synced_resources_returns_false(self):
 
990
        factory.make_BootResource(rtype=BOOT_RESOURCE_TYPE.UPLOADED)
 
991
        self.assertFalse(bootresources.has_synced_resources())
 
992
 
 
993
    def test__import_resources_exits_early_if_lock_held(self):
 
994
        has_synced_resources = self.patch_autospec(
 
995
            bootresources, "has_synced_resources")
 
996
        with transaction.atomic():
 
997
            with bootresources.locks.import_images:
 
998
                bootresources._import_resources(force=True)
 
999
        # The test for already-synced resources is not called if the
 
1000
        # lock is already held.
 
1001
        self.assertThat(has_synced_resources, MockNotCalled())
 
1002
 
 
1003
    def test__import_resources_exists_early_without_force(self):
 
1004
        has_synced_resources = self.patch(
 
1005
            bootresources, "has_synced_resources")
 
1006
        bootresources._import_resources(force=False)
 
1007
        # The test for already-synced resources is not performed if we're
 
1008
        # forcing a sync.
 
1009
        self.assertThat(has_synced_resources, MockCalledOnceWith())
 
1010
 
 
1011
    def test__import_resources_continues_with_force(self):
 
1012
        has_synced_resources = self.patch(
 
1013
            bootresources, "has_synced_resources")
 
1014
        bootresources._import_resources(force=True)
 
1015
        # The test for already-synced resources is performed if we're not
 
1016
        # forcing a sync.
 
1017
        self.assertThat(has_synced_resources, MockNotCalled())
 
1018
 
 
1019
    def test__import_resources_holds_lock(self):
 
1020
        fake_write_all_keyrings = self.patch(
 
1021
            bootresources, 'write_all_keyrings')
 
1022
 
 
1023
        def test_for_held_lock(directory, sources):
 
1024
            self.assertTrue(bootresources.locks.import_images.is_locked())
 
1025
            return []
 
1026
        fake_write_all_keyrings.side_effect = test_for_held_lock
 
1027
 
 
1028
        bootresources._import_resources(force=True)
 
1029
        self.assertFalse(bootresources.locks.import_images.is_locked())
 
1030
 
 
1031
    def test__import_resources_calls_functions_with_correct_parameters(self):
 
1032
        fake_write_all_keyrings = self.patch(
 
1033
            bootresources, 'write_all_keyrings')
 
1034
        fake_write_all_keyrings.return_value = sentinel.sources
 
1035
        fake_image_descriptions = self.patch(
 
1036
            bootresources, 'download_all_image_descriptions')
 
1037
        descriptions = Mock()
 
1038
        descriptions.is_empty.return_value = False
 
1039
        fake_image_descriptions.return_value = descriptions
 
1040
        fake_map_products = self.patch(
 
1041
            bootresources, 'map_products')
 
1042
        fake_map_products.return_value = sentinel.mapping
 
1043
        fake_download_all_boot_resources = self.patch(
 
1044
            bootresources, 'download_all_boot_resources')
 
1045
 
 
1046
        bootresources._import_resources(force=True)
 
1047
 
 
1048
        self.assertThat(
 
1049
            fake_write_all_keyrings,
 
1050
            MockCalledOnceWith(ANY, []))
 
1051
        self.assertThat(
 
1052
            fake_image_descriptions,
 
1053
            MockCalledOnceWith(sentinel.sources))
 
1054
        self.assertThat(
 
1055
            fake_map_products,
 
1056
            MockCalledOnceWith(descriptions))
 
1057
        self.assertThat(
 
1058
            fake_download_all_boot_resources,
 
1059
            MockCalledOnceWith(sentinel.sources, sentinel.mapping))
 
1060
 
 
1061
    def test__import_resources_has_env_GNUPGHOME_set(self):
 
1062
        fake_image_descriptions = self.patch(
 
1063
            bootresources, 'download_all_image_descriptions')
 
1064
        descriptions = Mock()
 
1065
        descriptions.is_empty.return_value = False
 
1066
        fake_image_descriptions.return_value = descriptions
 
1067
        self.patch(bootresources, 'map_products')
 
1068
        capture = self.patch_and_capture_env_for_download_all_boot_resources()
 
1069
 
 
1070
        bootresources._import_resources(force=True)
 
1071
        self.assertEqual(
 
1072
            bootresources.get_maas_user_gpghome(),
 
1073
            capture.env['GNUPGHOME'])
 
1074
 
 
1075
    def test__import_resources_has_env_http_and_https_proxy_set(self):
 
1076
        proxy_address = factory.make_name('proxy')
 
1077
        Config.objects.set_config('http_proxy', proxy_address)
 
1078
 
 
1079
        fake_image_descriptions = self.patch(
 
1080
            bootresources, 'download_all_image_descriptions')
 
1081
        descriptions = Mock()
 
1082
        descriptions.is_empty.return_value = False
 
1083
        fake_image_descriptions.return_value = descriptions
 
1084
        self.patch(bootresources, 'map_products')
 
1085
        capture = self.patch_and_capture_env_for_download_all_boot_resources()
 
1086
 
 
1087
        bootresources._import_resources(force=True)
 
1088
        self.assertEqual(
 
1089
            (proxy_address, proxy_address),
 
1090
            (capture.env['http_proxy'], capture.env['http_proxy']))
 
1091
 
 
1092
    def test__import_resources_calls_import_boot_images_on_clusters(self):
 
1093
        nodegroup = MagicMock()
 
1094
        self.patch(bootresources, 'NodeGroup', nodegroup)
 
1095
 
 
1096
        fake_image_descriptions = self.patch(
 
1097
            bootresources, 'download_all_image_descriptions')
 
1098
        descriptions = Mock()
 
1099
        descriptions.is_empty.return_value = False
 
1100
        fake_image_descriptions.return_value = descriptions
 
1101
        self.patch(bootresources, 'map_products')
 
1102
        self.patch(bootresources, 'download_all_boot_resources')
 
1103
 
 
1104
        bootresources._import_resources(force=True)
 
1105
        self.assertThat(
 
1106
            nodegroup.objects.import_boot_images_on_accepted_clusters,
 
1107
            MockCalledOnceWith())