~andreserl/maas/fix_pause_and_keep_running

import os
import tempfile
from contextlib import closing
from distutils.version import StrictVersion
from itertools import count
from textwrap import dedent
from time import sleep
from unittest import (
    skipIf,
    skipUnless,
)
try:
    from urllib2 import (
        urlopen,
        HTTPError,
    )
except ImportError:
    from urllib.request import urlopen
    from urllib.error import HTTPError

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

from apiclient.creds import convert_tuple_to_string

from netaddr import (
    IPAddress,
    IPRange,
)
from simplejson import loads
from testtools import TestCase
from testtools.content import (
    content_from_file,
    content_from_stream,
    text_content,
)
from testtools.matchers import (
    Contains,
    ContainsAll,
    ContainsDict,
    Equals,
    HasLength,
    Is,
    MatchesRegex,
    Not,
    MatchesDict,
    MatchesListwise,
)
from timeout import timeout
from config import (
    ADMIN_USER,
    ARM_LAB,
    BMC_START_IP,
    BMC_END_IP,
    DO_NOT_USE_ARM_NODES,
    GEN9_PASS,
    GEN9_SYSTEMS,
    GEN9_USER,
    GIGABYTE_SYSTEMS,
    GIGABYTE_USER,
    GIGABYTE_PASS,
    TEST_JUJU,
    TEST_CUSTOM_IMAGES,
    TEST_WINDOWS,
    HTTP_PROXY,
    LAB_DNS_CONFIG,
    LENOVO_LAB,
    MAAS_URL,
    MAIN_ARCHIVE,
    PASSWORD,
    PAUSE_CI,
    PORTS_ARCHIVE,
    POWER_PASS,
    POWER_USER,
    PPC_SYSTEMS,
    SLAVES,
    SQUID_DEB_PROXY_URL,
    USER_DATA_URL,
    USE_ARM_NODES,
    USE_PPC_NODES,
    WINDOWS_IMAGE,
    USE_ARM64_NODES,
)
from utils import (
    assertCommandReturnCode,
    get_config_value,
    get_maas_revision,
    get_maas_version,
    get_machines_with_status,
    get_ssh_key,
    get_ubuntu_version,
    pause_until_released,
    retries,
    run_command,
    run_maas_cli,
    setup_ssh,
    wait_machines,
)
import yaml


# Force Django to use MAAS's production settings.
if StrictVersion(get_maas_version()[:3]) > StrictVersion('2.1'):
    os.environ['DJANGO_SETTINGS_MODULE'] = 'maasserver.djangosettings.settings'
else:
    import sys
    sys.path.insert(0, "/usr/share/maas")
    os.environ['DJANGO_SETTINGS_MODULE'] = 'maas.settings'


# TODO: I think all this code can be removed - 2.0 allows the admin user to be
# created on the command line without any trickery - Brendan Donegan
import django  # noqa
django.setup()

from django.core.management import call_command  # noqa
from django.contrib.auth.models import User  # noqa

from maasserver.enum import NODE_STATUS  # noqa
from metadataserver.enum import RESULT_TYPE  # noqa
from maasserver.models.user import get_creds_tuple  # noqa


def setup_juju_cloud(server, cloud_yaml):
    cloud_yaml.write("""
clouds:
    testmaas:
        type: maas
        auth-types: [oauth1]
        endpoint: {server}""".format(server=server).encode('utf8'))
    cloud_yaml.file.flush()


def setup_juju_authentication(credentials_yaml, oauth):
    credentials_yaml.write("""
credentials:
    testmaas:
        test:
            auth-type: oauth1
            maas-oauth: {oauth}""".format(oauth=oauth))
    credentials_yaml.flush()


def get_token_str():
    admin = User.objects.get(username=ADMIN_USER)
    token = admin.tokens.all()[0]
    return convert_tuple_to_string(get_creds_tuple(token))


def setup_local_dns():
    with open('/etc/resolv.conf', encoding='ascii') as fd:
        content = fd.read()
    content = 'nameserver 127.0.0.1\n' + content
    with open('/etc/resolv.conf', mode='w', encoding='ascii') as f:
        f.write(content)


DEB_PROXY_CONFIG = """
cache_peer %s parent %s 0 no-query no-digest
never_direct allow all
""" % (urlparse(SQUID_DEB_PROXY_URL).hostname,
       urlparse(SQUID_DEB_PROXY_URL).port)


# The 'maas-cli' has been renamed to 'maas' in version 1931.
maascli = 'maas' if get_maas_revision() >= 1931 else 'maas-cli'

# The 'maas-region-admin' has been renamed to 'maas-region' in version .
maas_region_admin = (
    'maas-region' if get_maas_revision() >= 4738 else 'maas-region-admin')

# Uses the pre-1.6 API, where a cluster interface had no "name" field.
# MAAS will take the name from the "interface" field, as long as it's
# a unique name within the cluster.  Newer clients would pass a name.
REGION_DHCP_CONFIG = {
    "ip": "10.245.136.6",
    "interface": "eth1",
    "subnet_mask": "255.255.248.0",
    "broadcast_ip": "10.245.143.255",
    "router_ip": "10.245.136.1",
    "management": 2,
    "ip_range_low": "10.245.136.10",
    "ip_range_high": "10.245.136.200",
    # add cidr as it is now default in new networking configs.
    "cidr": "10.245.136.0/21"
}


class TestMAASIntegration(TestCase):

    def get_node_count(self):
        """The number of available nodes."""
        node_count = len(LENOVO_LAB) + len(GEN9_SYSTEMS)
        if USE_ARM64_NODES:
            node_count += len(GIGABYTE_SYSTEMS)
        if USE_ARM_NODES:
            node_count += len(ARM_LAB)
        if USE_PPC_NODES:
            node_count += len(PPC_SYSTEMS)
        return node_count

    def _wait_maas_running(self):
        for _ in range(10):
            try:
                urlopen(MAAS_URL).code
            except HTTPError:
                sleep(30)
                continue
            return True
        return False

    def _run_maas_cli(self, args):
        return run_maas_cli(self, "maas", args)

    def _update_dhcpd_apparmor_profile(self):
        """Workaround for raring due to bug 1107686."""
        with open("/etc/apparmor.d/usr.sbin.dhcpd", "r+") as dhcpd_fd:
            dhcpd_file = dhcpd_fd.read().decode('ascii')
            dhcpd_file = dhcpd_file.replace(
                'network packet packet,',
                'network packet packet,\n  network packet raw,')
            dhcpd_fd.seek(0)
            dhcpd_fd.write(dhcpd_file.encode('utf-8'))
        cmd = ["service", "apparmor", "reload"]
        expected_output = '* Reloading AppArmor profiles'
        assertCommandReturnCode(self, cmd, expected_output)

    def _get_rack_interface_link(self, primary_rack_system_id, ip):
        rack, _ = self._run_maas_cli(
            ['rack-controller', 'read', primary_rack_system_id])
        rack_controller = loads(rack)

        for interface in rack_controller['interface_set']:
            for link in interface['links']:
                if ip == link['ip_address']:
                    return link
        return None

    def _set_up_dhcp(self, primary_rack_system_id, ip):
        # Support the two following cases:
        #  1. eth1 is auto-detected
        #  2. eth1  not auto-dectected. in 2.0 this would be a bug.
        rack_interface_link = self._get_rack_interface_link(
            primary_rack_system_id, ip
        )
        rack_fabric = rack_interface_link['subnet']['vlan']['fabric']
        rack_vlan = rack_interface_link['subnet']['vlan']['name']

        maas_dhcp_cmd = [
            'vlan', 'update', rack_fabric, rack_vlan, 'dhcp_on=True',
            'primary_rack=%s' % primary_rack_system_id
        ]
        output, _ = self._run_maas_cli(maas_dhcp_cmd)
        vlan_result = loads(output)
        self.assertEqual(True, vlan_result['dhcp_on'])

    def _boot_nodes(self):
        # Run ipmipower to boot up nodes.
        for ipmi_address in LENOVO_LAB.values():
            self.cycle_power(
                ipmi_address, user=POWER_USER, password=POWER_PASS
            )
        for ipmi_address in GEN9_SYSTEMS.values():
            self.cycle_power(
                ipmi_address, driver='LAN_2_0',
                user=GEN9_USER, password=GEN9_PASS
            )
        if USE_ARM64_NODES:
            for ipmi_address in GIGABYTE_SYSTEMS.values():
                self.cycle_power(
                    ipmi_address, user=GIGABYTE_USER, password=GIGABYTE_PASS
                )
        if USE_PPC_NODES:
            for ipmi_address in PPC_SYSTEMS.values():
                self.cycle_power(
                    ipmi_address, password='admin', driver='LAN_2_0'
                )
        if USE_ARM_NODES:
            for ipmi_address in ARM_LAB.values():
                self.cycle_power(ipmi_address, user='admin', password='admin')

    def cycle_power(self, ip, user=None, password=None, driver=None):
        self.power_off(ip, user=user, password=password, driver=driver)
        cmd = ["ipmi-chassis-config", "-h", ip]
        if driver is not None:
            cmd += ["-D", driver]
        if user is not None:
            cmd += ["-u", user]
        if password is not None:
            cmd += ["-p", password]
        config_cmd = cmd + ["--config", "--file=ipmi.conf"]
        run_command(config_cmd)
        self.power_on(ip, user=user, password=password, driver=driver)

    def _issue_power_command(self, state, ip,
                             user=None, password=None, driver=None):
        cmd = ["ipmipower", "-h", ip]
        if driver is not None:
            cmd += ["-D", driver]
        if user is not None:
            cmd += ["-u", user]
        if password is not None:
            cmd += ["-p", password]
        power_cmd = cmd + ["--%s" % state]
        power_stat_cmd = cmd + ["--stat"]
        expected_stat = "%s: %s" % (ip, state)
        for _ in retries(delay=5, timeout=30):
            retcode, output, err = run_command(power_cmd)
            # We don't check the output as it can be misleading, e.g.
            # HP Gen9's can return empty output on success
            self.addDetail('%s stdout' % cmd, text_content(output))
            self.addDetail('%s stderr' % cmd, text_content(err))
            self.assertIs(retcode, 0)
            _, output, _ = run_command(power_stat_cmd)
            if expected_stat in output:
                break

    def power_on(self, ip, user=None, password=None, driver=None):
        self._issue_power_command(
            'on', ip, user=user, password=password, driver=driver,
        )

    def power_off(self, ip, user=None, password=None, driver=None):
        self._issue_power_command(
            'off', ip, user=user, password=password, driver=driver,
        )

    def _wait_machines(self, status=None, num_expected=None):
        """Wait for `num_expected` nodes with status `status`."""
        if num_expected is None:
            num_expected = self.get_node_count()
        return wait_machines(self, num_expected, status)

    def _run_juju_command(self, args, env=None):
        command = ["juju", "--debug"]
        command.extend(args)
        retcode, output, err = run_command(command, env=env)
        command_name = " ".join(command)
        self.addDetail(command_name, text_content(output))
        self.addDetail(command_name, text_content(err))
        return retcode, output, err

    def get_juju_status(self):
        # Juju2 status defaults to tabular so we need to force YAML
        _, status_output, _ = self._run_juju_command(
            ["status", "--format=yaml"]
        )
        status = yaml.safe_load(status_output)
        return status

    def _wait_machines_running(self, nb_machines):
        """Wait until at least `nb_machines` have their agent running."""
        while True:
            status = self.get_juju_status()
            if status is not None:
                machines = status['machines'].values()
                running_machines = [
                    m for m in machines if
                    m.get('machine-status', {}).get('current', '') in ['running', 'started']  # noqa
                ]
                if len(running_machines) >= nb_machines:
                    break
            sleep(20)

    def _wait_units_started(self, application, nb_units):
        """Wait until an application has at least `nb_units` units."""
        while True:
            status = self.get_juju_status()
            try:
                units = status['applications'][application]['units'].values()
            except KeyError:
                units = []
            started_units = [
                unit for unit in units
                if unit.get('workload-status', {}).get('current', '') == 'active'  # noqa
            ]
            if len(started_units) >= nb_units:
                return started_units
            sleep(20)

    def _rack_imported_images(self):
        rack_system_id = self._get_rack_systemid_on_region()
        while True:
            output, _ = self._run_maas_cli(
                ["rack-controller", "list-boot-images", rack_system_id])
            rack_images = loads(output)
            if rack_images['status'] == 'synced':
                break
            self.addDetail(
                "Waiting for rack to import boot resources.",
                text_content(output))
            sleep(5)

    def test_create_admin(self):
        """Run maas createsuperuser."""
        cmd_output = call_command(
            "createadmin", username=ADMIN_USER, password=PASSWORD,
            email="example@canonical.com", noinput=True)
        self.assertEqual(cmd_output, None)

    # Since revision 1828, MAAS doesn't use avahi/dbus anymore so
    # we do not need to restart these daemons.
    @skipIf(
        get_maas_revision() >= 1828, "Avahi/DBUS are not used anymore")
    def test_restart_dbus_avahi(self):
        cmd = ["service", "dbus", "restart"]
        expected_output = 'dbus start/running'
        assertCommandReturnCode(self, cmd, expected_output)
        cmd = ["service", "avahi-daemon", "restart"]
        expected_output = 'avahi-daemon start/running'
        assertCommandReturnCode(self, cmd, expected_output)

    @skipIf(
        os.path.exists("/etc/maas/maas_local_settings.py"),
        "local_config_set is not available")
    def test_update_maas_url(self):
        # XXX: allenap: Using a debconf file to set the initial value of the
        # MAAS URL should not work; see test_update_maas_url_old(), but I'm
        # not sure if that's true.
        cmd = [
            "maas-region", "local_config_set", "--maas-url",
            "http://10.245.136.6/MAAS",
        ]
        assertCommandReturnCode(self, cmd, "")
        # Restart apache2.
        cmd = ["systemctl", "restart", "maas-regiond"]
        for _ in retries(delay=10, timeout=60):
            retcode, _, _ = run_command(cmd)
            if retcode == 0:
                break
        else:
            self.fail("maas-regiond cannot be restarted")

    def test_update_maas_url_rack(self):
        cmd = [
            "maas-rack", "config", "--region-url",
            "http://10.245.136.6:5240/MAAS",
        ]
        assertCommandReturnCode(self, cmd, "")
        # Restart apache2.
        cmd = ["systemctl", "restart", "maas-rackd"]
        for _ in retries(delay=10, timeout=60):
            retcode, _, _ = run_command(cmd)
            if retcode == 0:
                break
        else:
            self.fail("maas-rackd could not be restarted")

    @skipIf(
        StrictVersion(get_ubuntu_version()) < StrictVersion('15.10'),
        'From 15.10 onwards, we use systemd')
    def test_check_initial_services_systemctl(self):
        cmd = ["systemctl", "status", "maas-rackd"]
        retcode, output, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertIn('Active: active (running)', output)
        self.assertEqual('', err)

    def test_check_rpc_info(self):
        # Ensure that the region is publishing RPC info to the clusters. This
        # is a reasonable indication that the region is running correctly.
        url = "%s/rpc/" % MAAS_URL.rstrip("/")
        self.addDetail('url', text_content(url))
        errors = []
        attempts = count(1)
        for elapsed, _ in retries(timeout=300, delay=10):
            attempt = next(attempts)
            try:
                with closing(urlopen(url)) as response:
                    details = content_from_stream(response, buffer_now=True)
            except HTTPError as error:
                errors.append(error)
            else:
                self.addDetail('info', details)
                break
        else:
            self.addDetail('errors', text_content(
                "\n--\n".join([err for err in errors])))
            self.fail(
                "RPC info not published after %d attempts over %d seconds."
                % (attempt, elapsed))

    @skipIf(DO_NOT_USE_ARM_NODES, "Don't test ARM nodes")
    def test_update_preseed_arm(self):
        # XXX: matsubara add workaround to boot arm nodes.
        try:
            # Try the old location first, the one used before the templates
            # were moved to /etc/maas/.
            userdata_fd = open(
                "/usr/share/maas/preseeds/enlist_userdata", "rb+")
        except IOError:
            userdata_fd = open("/etc/maas/preseeds/enlist_userdata", "rb+")
        userdata = userdata_fd.read().decode('utf-8')
        url = 'http://ports.ubuntu.com/ubuntu-ports'
        userdata += '\n' + dedent("""\
            apt_sources:
            - source: "deb %s precise-proposed main"
            """) % url
        userdata_fd.seek(0)
        userdata_fd.write(userdata.encode('utf-8'))
        userdata_fd.close()

    def test_login_api(self):
        self.assertTrue(self._wait_maas_running())
        token_str = get_token_str()
        api_url = MAAS_URL + "/api/2.0/"
        cmd = [maascli, "login", "maas", api_url, token_str]
        expected_output = "\nYou are now logged in to the MAAS server"
        for _ in range(30):
            retcode, output, err = run_command(cmd, env=os.environ.copy())
            if retcode == 0:
                break
            elif retcode == 1 and 'Expected application/json' in err:
                # The cli may error with 'Expected application/json, got:
                # text/html; charset=utf-8\n' with retcode 1 before all pieces
                # are up and running.
                sleep(10)  # We need to give MAAS some time.
        self.assertIn(expected_output, output)

    def test_maas_logged_in(self):
        """Make sure MAAS is successfully logged in. Also, it signals a handy
        place to put a pause."""
        cmd = ["maas", "list"]
        retcode, _, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertEqual('', err)

    def test_set_main_archive(self):
        output, _ = self._run_maas_cli([
            "maas", "set-config", "name=main_archive",
            "value=%s" % MAIN_ARCHIVE])
        self.assertThat(output, Contains("OK"))
        output, _ = self._run_maas_cli([
            "maas", "get-config", "name=main_archive"])
        self.assertThat(output, Contains(MAIN_ARCHIVE))

    @skipIf(
        StrictVersion(get_maas_version()[:3]) > StrictVersion('2.0'),
        'Cloud-init "package_mirror" config is used in MAAS 2.0 and earlier')
    def test_main_archive_in_enlist_userdata_package_mirrors_config(self):
        # Check that the user-data passed on to nodes have
        # the 'primary' mirror address.
        output = urlopen(USER_DATA_URL).read().decode('utf-8')
        self.assertThat(output, Contains(
            'primary:  ["%s"]' % MAIN_ARCHIVE))

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('2.1'),
        'Cloud-init "apt" config is used solely in MAAS 2.1+')
    def test_main_archive_in_enlist_userdata_apt_config(self):
        output = urlopen(USER_DATA_URL).read().decode('utf-8')
        # Old APT configuration.
        self.assertThat(yaml.safe_load(output), ContainsDict({
            'system_info': ContainsDict({
                'package_mirrors': MatchesListwise([
                    MatchesDict({
                        "arches": Equals(["i386", "amd64"]),
                        "search": MatchesDict({
                            "primary": Equals([MAIN_ARCHIVE]),
                            "security": Equals([MAIN_ARCHIVE]),
                            }),
                        "failsafe": MatchesDict({
                            "primary": Equals(
                                "http://archive.ubuntu.com/ubuntu"),
                            "security": Equals(
                                "http://security.ubuntu.com/ubuntu"),
                            })
                        }),
                    MatchesDict({
                        "arches": Equals(["default"]),
                        "search": MatchesDict({
                            "primary": Equals([PORTS_ARCHIVE]),
                            "security": Equals([PORTS_ARCHIVE]),
                            }),
                        "failsafe": MatchesDict({
                            "primary": Equals(
                                "http://ports.ubuntu.com/ubuntu-ports"),
                            "security": Equals(
                                "http://ports.ubuntu.com/ubuntu-ports"),
                            })
                        }),
                    ]),
                }),
            }))
        # New APT configuration.
        self.assertThat(yaml.safe_load(output), ContainsDict({
            'apt': ContainsDict({
                'preserve_sources_list': Equals(False),
                'primary': MatchesListwise([
                    MatchesDict({
                        "arches": Equals(["amd64", "i386"]),
                        "uri": Equals(MAIN_ARCHIVE),
                    }),
                    MatchesDict({
                        "arches": Equals(["default"]),
                        "uri": Equals(PORTS_ARCHIVE),
                    }),
                ]),
            })
        }))

    def test_set_http_proxy(self):
        output, _ = self._run_maas_cli([
            "maas", "set-config", "name=http_proxy",
            "value=%s" % HTTP_PROXY])
        self.assertThat(output, Contains("OK"))
        output, _ = self._run_maas_cli([
            "maas", "get-config", "name=http_proxy"])
        self.assertThat(output, Contains(HTTP_PROXY))

        # Check that the user-data passed on to nodes have the proxy address.
        output = urlopen(USER_DATA_URL).read().decode('utf-8')
        self.assertThat(output, Contains(
            "apt_proxy: %s" % HTTP_PROXY))

    def get_master_rack(self):
        output, _ = self._run_maas_cli(["rack-controllers", "read"])
        rack_controllers = loads(output)
        if len(rack_controllers) > 0:
            return rack_controllers[0]
        return None

    def get_rack_service_status(self, rack_id, service_name):
        output, _ = self._run_maas_cli(["rack-controller", "read", rack_id])
        rack_info = loads(output)
        for service in rack_info['service_set']:
            if service['name'] == service_name:
                return service['status']
        return ''

    @timeout(5 * 60)
    def test_region_rack_connected(self):
        # The master cluster is connected and changed the uuid field of the
        # nodegroup object from 'master' to its UUID.
        while self.get_master_rack() is None:
            sleep(10)
        rack_id = self.get_master_rack()['system_id']
        while self.get_rack_service_status(rack_id, 'rackd') != 'running':
            sleep(10)

    def test_create_dynamic_range(self):
        output, _ = self._run_maas_cli(
            ['ipranges', 'create', 'type=dynamic',
             'start_ip=%s' % REGION_DHCP_CONFIG['ip_range_low'],
             'end_ip=%s' % REGION_DHCP_CONFIG['ip_range_high']])
        # If the range was configured correct, it should return the subnet
        # where the range belogs to. As such, we check the output to ensure
        # it is there.
        self.assertThat(
            output, Contains('"cidr": "%s"' % REGION_DHCP_CONFIG['cidr'])
        )
        # TODO: We need to read all IP ranges for the subnet and check that
        # the range has actually been created regardless whether we checked
        # the output above.
        _, _ = self._run_maas_cli(
            ['ipranges', 'read'])

    def _get_rack_systemid_on_region(self):
        output, _ = self._run_maas_cli(["rack-controllers", "read"])
        rack_controllers = loads(output)
        region_rack = 4
        for rack in rack_controllers:
            if rack['node_type'] == region_rack:
                return rack['system_id']

    def test_create_slave_device_and_link_subnet(self):
        rack_system_id = self._get_rack_systemid_on_region()
        rack_interface_link = self._get_rack_interface_link(
            rack_system_id, REGION_DHCP_CONFIG['ip']
        )
        subnet = rack_interface_link['subnet']['name']
        # Create a device with the same IP as the MAAS CI slave so
        # that the IP address cannot be 'stolen' by another machine
        for slave in SLAVES:
            output, _ = self._run_maas_cli([
                "devices", "create", "hostname={}".format(slave['hostname']),
                "mac_addresses={}".format(slave["mac"])
            ])
            device = loads(output)
            device_id = device["system_id"]
            interface_id = "{}".format(device["interface_set"][0]["id"])
            # Update the vlan on the interface so we can link the subnet
            output, _ = self._run_maas_cli([
                "interface", "link-subnet", device_id, interface_id,
                "mode=STATIC", "subnet={}".format(subnet),
                "ip_address={}".format(slave["ip"]),
            ])
            linked_interface = loads(output)
            self.assertThat(
                linked_interface['links'][0]['ip_address'],
                Equals(slave["ip"])
            )

    def test_slave_device_interface_linked(self):
        """
        Check that after we created the slave device, that the
        representation in MAAS looks sensible.
        """
        output, _ = self._run_maas_cli(["devices", "read"])
        devices = loads(output)
        slave_hostnames = [slave['hostname'] for slave in SLAVES]
        slave_devices = [
            device for device in devices
            if device['hostname'] in slave_hostnames
        ]
        self.assertThat(len(slave_devices), Equals(len(SLAVES)))
        device_ips = []
        device_macs = []
        for device in slave_devices:
            device_ips.extend(device['ip_addresses'])
            macs = [iface['mac_address'].lower()
                    for iface in device['interface_set']]
            device_macs.extend(macs)
        slave_ips = [slave["ip"] for slave in SLAVES]
        slave_macs = [slave["mac"].lower() for slave in SLAVES]
        self.assertThat(device_ips, ContainsAll(slave_ips))
        self.assertThat(device_macs, ContainsAll(slave_macs))

    @timeout(10 * 60)
    def test_set_up_dhcp_vlan(self):
        rack_system_id = self._get_rack_systemid_on_region()
        self._set_up_dhcp(rack_system_id, REGION_DHCP_CONFIG['ip'])

        # Wait for the task to complete and create the dhcpd.conf file.
        while os.path.exists("/var/lib/maas/dhcpd.conf") is False:
            self.addDetail(
                "Waiting task create dhcpd.conf file.",
                content_from_file("/var/log/maas/maas.log"))
            sleep(2)

    @skipIf(
        StrictVersion(get_ubuntu_version()) < StrictVersion('15.10'),
        'From 15.10 onwards, we use systemd')
    # Timeout after 2 minutes, provided that it may take a while for the
    # rack controller to write dhcp config and start maas-dhcpd
    @timeout(2 * 60)
    def test_check_dhcp_service_systemctl(self):
        cmd = ["systemctl", "status", "maas-dhcpd"]
        # systemd will return 3 if a 'condition failed. This typically means
        # that /var/lib/maas/dhcpd.conf is not there yet, and we should wait
        # for a bit to see if the config is written and maas-dhcp is brought up
        # by the rack.
        retcode, output, err = run_command(cmd)
        while retcode != 0:
            # query systemd every 3 seconds to see if maas-dhcpd us running
            sleep(3)
            retcode, output, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertIn('Active: active (running)', output)
        self.assertEqual('', err)

    @skipIf(
        StrictVersion(get_ubuntu_version()) < StrictVersion('15.10'),
        'From 15.10 onwards, we use systemd')
    def test_update_dns_config_systemctl(self):
        dns_config = open("/etc/bind/named.conf.options", 'w')
        dns_config.write(LAB_DNS_CONFIG)
        dns_config.close()
        cmd = ["systemctl", "restart", "bind9"]
        retcode, output, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertEqual('', output)  # No news is good news
        self.assertEqual('', err)

    # Zones
    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_add_new_zones(self):
        # Create 2 new zones.
        output, _ = self._run_maas_cli(
            ["zones", "create", "name=test-zone", "description='A test zone'"])
        zone = loads(output)
        self.assertEqual('test-zone', zone['name'])
        output, _ = self._run_maas_cli(
            ["zones", "create", "name=delete-zone",
             "description='A test zone to be deleted'"])
        zone = loads(output)
        self.assertEqual('delete-zone', zone['name'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_list_zones(self):
        output, _ = self._run_maas_cli(["zones", "read"])
        zones = loads(output)
        expected = [u'default', u'delete-zone', u'test-zone']
        self.assertEqual(expected, sorted([zone['name'] for zone in zones]))

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_delete_zone(self):
        self._run_maas_cli(["zone", "delete", "delete-zone"])
        # List the remaining zones after the delete command.
        output, _ = self._run_maas_cli(["zones", "read"])
        zones = loads(output)
        self.assertNotIn(
            'delete-zone', sorted([zone['name'] for zone in zones]))

    # Spaces and subnets
    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Space feature only available after 1.9')
    def test_add_new_spaces(self):
        # Create 2 new spaces.
        output, _ = self._run_maas_cli(
            ['spaces', 'create', 'name=test-space'])
        out_dict = loads(output)
        self.assertEqual('test-space', out_dict['name'])
        output, _ = self._run_maas_cli(
            ['spaces', 'create', 'name=delete-space']
        )
        delete_space = loads(output)
        self.assertEqual('delete-space', delete_space['name'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Subnet feature only available after 1.9')
    def test_create_subnet(self):
        create_cmd = ['subnets', 'create', 'name=test-subnet',
                      'cidr=192.168.200.0/24']
        if StrictVersion(get_maas_version()[:3]) <= StrictVersion('2.1'):
            create_cmd.append('space=0')
        output, _ = self._run_maas_cli(create_cmd)
        out_dict = loads(output)
        self.assertEqual('test-subnet', out_dict['name'])
        self.assertEqual('192.168.200.0/24', out_dict['cidr'])
        if StrictVersion(get_maas_version()[:3]) <= StrictVersion('2.1'):
            self.assertEqual('space-0', out_dict['space'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Space feature only available after 1.9')
    def test_list_spaces(self):
        output, _ = self._run_maas_cli(['spaces', 'read'])
        out_dict = loads(output)
        if StrictVersion(get_maas_version()[:3]) <= StrictVersion('2.1'):
            expected = ['delete-space', 'space-0', 'test-space']
        else:
            expected = ['delete-space', 'test-space', 'undefined']
        self.assertItemsEqual(
            expected, [item['name'] for item in out_dict])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Subnet feature only available after 1.9')
    def test_list_subnets(self):
        output, _ = self._run_maas_cli(['subnets', 'read'])
        out_dict = loads(output)
        self.assertEqual(2, len(out_dict), 'space-0 should now have 2 subnets')

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Subnet feature only available after 1.9')
    def test_delete_subnet(self):
        self._run_maas_cli(['subnet', 'delete', 'test-subnet'])
        output, _ = self._run_maas_cli(['subnets', 'read'])
        subnets = loads(output)
        self.assertEqual(1, len(subnets), 'space-0 should now have 1 subnet')
        self.assertNotIn('test-subnet', [subnet['name'] for subnet in subnets])
        self.assertEqual(
            subnets[0]['name'], subnets[0]['cidr'],
            'Name and CIDR should be equal for the default subnet.')

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Space feature only available after 1.9')
    def test_delete_space(self):
        self._run_maas_cli(['space', 'delete', 'delete-space'])
        # List the remaining zones after the delete command.
        output, _ = self._run_maas_cli(['spaces', 'read'])
        out_dict = loads(output)
        self.assertNotIn(
            'delete-space', [item['name'] for item in out_dict])

    # Fabrics and VLANs
    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Fabric feature only available after 1.9')
    def test_add_new_fabrics(self):
        # Create 2 new fabrics.
        output, _ = self._run_maas_cli(
            ['fabrics', 'create', 'name=test-fabric'])
        test_fabric = loads(output)
        self.assertEqual('test-fabric', test_fabric['name'])
        output, _ = self._run_maas_cli(
            ['fabrics', 'create', 'name=delete-fabric'])
        delete_fabric = loads(output)
        self.assertEqual('delete-fabric', delete_fabric['name'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'VLAN feature only available after 1.9')
    def test_add_vlan_to_fabric(self):
        output, _ = self._run_maas_cli(
            ['vlans', 'create', 'test-fabric', 'name=test-vlan', 'vid=2'])
        test_vlan = loads(output)
        self.assertEqual('test-vlan', test_vlan['name'])
        self.assertEqual('test-fabric', test_vlan['fabric'])
        self.assertEqual(2, test_vlan['vid'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Fabric feature only available after 1.9')
    def test_list_fabrics(self):
        output, _ = self._run_maas_cli(['fabrics', 'read'])
        fabrics = loads(output)
        fabric_names = [fabric['name'] for fabric in fabrics]
        self.assertThat(fabric_names, Contains('test-fabric'))
        fabric_names.remove('test-fabric')
        self.assertThat(fabric_names, Contains('delete-fabric'))
        fabric_names.remove('delete-fabric')
        for fabric_name in fabric_names:
            self.assertThat(fabric_name, MatchesRegex(r'fabric-\d$'))
        self.assertIn(
            'test-vlan',
            [[v['name'] for v in f['vlans']]
             for f in fabrics if f['name'] == 'test-fabric'][0])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'VLAN feature only available after 1.9')
    def test_list_vlans(self):
        output, _ = self._run_maas_cli(['vlans', 'read', 'test-fabric'])
        vlans = loads(output)
        expected = ['untagged', 'test-vlan']
        self.assertItemsEqual(expected, [v['name'] for v in vlans])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'VLAN feature only available after 1.9')
    def test_delete_vlan(self):
        self._run_maas_cli(['vlan', 'delete', 'test-fabric', 'test-vlan'])
        output, _ = self._run_maas_cli(['vlans', 'read', 'test-fabric'])
        vlans = loads(output)
        self.assertNotIn('test-vlan', [v['name'] for v in vlans])
        self.assertEqual(
            1, len(vlans), 'Fabric should have only one VLAN now.'
        )

    def test_reserve_bmc_range(self):
        self._run_maas_cli([
            'ipranges', 'create', 'type=reserved',
            'start_ip=' + BMC_START_IP, 'end_ip=' + BMC_END_IP,
            'comment=BMCs'
        ])
        output, _ = self._run_maas_cli(
            ['ipranges', 'read']
        )
        ipranges = loads(output)
        # TODO: assert that newly created range is there
        bmc_range = None
        for iprange in ipranges:
            if iprange['comment'] == 'BMCs':
                bmc_range = iprange
        self.assertIsNotNone(bmc_range, 'BMC range not found')
        self.assertThat(bmc_range['start_ip'], Equals(BMC_START_IP))
        self.assertThat(bmc_range['end_ip'], Equals(BMC_END_IP))
        self.assertThat(bmc_range['type'], Equals('reserved'))

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        "Fabric feature only available after 1.9")
    def test_delete_fabric(self):
        self._run_maas_cli(["fabric", "delete", "delete-fabric"])
        # List the remaining zones after the delete command.
        output, _ = self._run_maas_cli(["fabrics", "read"])
        fabrics = loads(output)
        self.assertNotIn(
            'delete-fabric', sorted([fabric['name'] for fabric in fabrics]))

    @skipUnless(TEST_CUSTOM_IMAGES, "Not testing custom images")
    def test_set_custom_boot_source(self):
        boot_source_url = "http://10.55.32.203/proposed/streams/v1/index.json"
        output, _ = self._run_maas_cli([
            "boot-source", "update", "1",
            "url=" + boot_source_url,
        ])
        boot_source = loads(output)
        self.assertThat(boot_source['url'], Equals(boot_source_url))

    @timeout(60 * 30)
    def test_stop_image_import(self):
        while True:
            output, _ = self._run_maas_cli(
                ["boot-resources", "is-importing"]
            )
            importing = loads(output)
            if not importing:
                break

    @skipUnless(USE_PPC_NODES, "Not testing PPC systems")
    def test_add_boot_source_selection_ppc64el(self):
        # Add the ppc64el boot source selection to all boot sources
        output, _ = self._run_maas_cli(["boot-sources", "read"])
        boot_sources = loads(output)
        for source in boot_sources:
            output, _ = self._run_maas_cli(
                ["boot-source-selections", "read", str(source["id"])]
            )
            # To add a new arch we need to specify the arches= parameter
            # multiple times, e.g. arches=amd64 arches=ppc64el
            for selection in loads(output):
                new_arches = selection['arches'] + ['ppc64el']
                selection_update_params = [
                    'arches={arch}'.format(arch=arch) for arch in new_arches
                ]
                output, _ = self._run_maas_cli([
                    "boot-source-selection",
                    "update",
                    str(source["id"]),
                    str(selection["id"]),
                ] + selection_update_params)
                updated_selection = loads(output)
                self.assertThat(
                    updated_selection["arches"], Equals(new_arches)
                )

    @skipUnless(USE_ARM64_NODES, "Not testing arm64 systems")
    def test_add_boot_source_selection_arm64(self):
        # Add the arm64 boot source selection to all boot sources
        output, _ = self._run_maas_cli(["boot-sources", "read"])
        boot_sources = loads(output)
        for source in boot_sources:
            output, _ = self._run_maas_cli(
                ["boot-source-selections", "read", str(source["id"])]
            )
            # To add a new arch we need to specify the arches= parameter
            # multiple times, e.g. arches=amd64 arches=amd64
            for selection in loads(output):
                new_arches = selection['arches'] + ['arm64']
                selection_update_params = [
                    'arches={arch}'.format(arch=arch) for arch in new_arches
                ]
                output, _ = self._run_maas_cli([
                    "boot-source-selection",
                    "update",
                    str(source["id"]),
                    str(selection["id"]),
                ] + selection_update_params)
                updated_selection = loads(output)
                self.assertThat(
                    updated_selection["arches"], Equals(new_arches)
                )

    def test_add_boot_source_selection_centos(self):
        output, _ = self._run_maas_cli(["boot-sources", "read"])
        boot_sources = loads(output)
        for source in boot_sources:
            output, _ = self._run_maas_cli([
                "boot-source-selections",
                "create",
                "%d" % source["id"],
                "os=centos",
                "release=centos70",
                "subarches=*",
                "arches=amd64",
                "labels=*",
            ])

    @timeout(30*60)
    @skipUnless(TEST_WINDOWS, "Not testing Windows")
    def test_add_windows_boot_resource(self):
        # Upload the Windows image as a custom boot source
        _, _ = self._run_maas_cli([
            "boot-resources",
            "create",
            "name=windows/win2012hvr2",
            "title=Windows2012HVR2",
            "architecture=amd64/generic",
            "filetype=ddtgz",
            "content@=" + WINDOWS_IMAGE,
        ])
        expected_resources = set([
            ('windows/win2012hvr2', 'amd64/generic'),
        ])
        self._region_imported_images(expected_resources)
        self._rack_imported_images()

    def test_start_image_import(self):
        for _ in retries(delay=10, timeout=60):
            output, _ = self._run_maas_cli(
                ["boot-resources", "is-importing"]
            )
            importing = loads(output)
            if importing:
                break
            # Start the import if it is currently stopped
            output, _ = self._run_maas_cli(
                ["boot-resources", "import"]
            )
            self.assertThat(
                output, Contains(
                    "Import of boot resources started"
                )
            )

    def _region_imported_images(self, expected_resources):
        complete_resources = set()
        while not expected_resources.issubset(complete_resources):
            resources_output, _ = self._run_maas_cli(
                ["boot-resources", "read"])
            resources = loads(resources_output)
            for resource in resources:
                resource_id = resource['id']
                resource_name = resource['name']
                resource_arch = resource['architecture']
                output, _ = self._run_maas_cli(
                    ['boot-resource', 'read', '%s' % resource_id])
                resource_data = loads(output)
                for _, resource_set in resource_data['sets'].items():
                    if resource_set['complete']:
                        complete_resources.add((resource_name, resource_arch))
            self.addDetail(
                "Waiting for region to import boot resources.",
                text_content(resources_output))
            sleep(5)

    @timeout(60 * 60)  # Allow for up to one hour
    def test_region_imported_images(self):
        osystem_series = '{}/{}'.format(
            get_config_value(self, 'default_osystem'),
            get_config_value(self, 'default_distro_series'),
        )
        # MAAS 2.1 and up uses new kernel naming scheme
        if StrictVersion(get_maas_version()[:3]) > StrictVersion('2.0'):
            expected_resources = set([
                (osystem_series, 'amd64/hwe-16.04'),
                (osystem_series, 'amd64/hwe-16.04-lowlatency'),
                (osystem_series, 'amd64/ga-16.04'),
                (osystem_series, 'amd64/ga-16.04-lowlatency'),
                (u'centos/centos70', 'amd64/generic'),
            ])
            if USE_ARM64_NODES:
                expected_resources.update([
                    (osystem_series, 'arm64/ga-16.04'),
                    (osystem_series, 'arm64/hwe-16.04'),
                    (osystem_series, 'arm64/xgene-uboot'),
                    (osystem_series, 'arm64/xgene-uboot-mustang'),
                ])
            if USE_PPC_NODES:
                expected_resources.update([
                    (osystem_series, 'ppc64el/hwe-16.04'),
                    (osystem_series, 'ppc64el/ga-16.04'),
                ])
        else:
            expected_resources = set([
                (osystem_series, 'amd64/hwe-x'),
            ])
            if USE_ARM64_NODES:
                expected_resources.update([
                    (osystem_series, 'arm64/hwe-x'),
                ])
            if USE_PPC_NODES:
                expected_resources.update([
                    (osystem_series, 'ppc64el/hwe-x'),
                ])
        self._region_imported_images(expected_resources)

    @timeout(60 * 30)  # Allow for up to 30 mins
    def test_rack_imported_images(self):
        self._rack_imported_images()

    def test_add_ssh_key(self):
        """Add our ssh key to the admin account."""
        setup_ssh()
        ssh_key = get_ssh_key()
        self._run_maas_cli(["sshkeys", "create", "key=%s" % ssh_key])
        out, _ = self._run_maas_cli(["sshkeys", "read"])
        keys = loads(out)
        self.assertEqual(ssh_key, keys[0]['key'])

    def test_poweron_nodes_to_enlist(self):
        self._boot_nodes()
        if PAUSE_CI:
            pause_until_released("Powering nodes for enlistment and pausing CI")

    @timeout(60 * 30)
    def test_check_machines_new(self):
        self._wait_machines(NODE_STATUS.NEW)  # 0

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_assign_machines_to_test_zone(self):
        # Set two of the declared nodes to the test-zone created earlier.
        output, _ = self._run_maas_cli(["machines", "read"])
        machines = loads(output)
        for machine in machines[:2]:
            self._run_maas_cli(
                ["machine", "update", machine['system_id'], "zone=test-zone"])
        # Check nodes are in the test-zone
        output, _ = self._run_maas_cli(["machines", "read", "zone=test-zone"])
        machines = loads(output)
        self.assertEqual(2, len(machines))

    def test_set_machines_ipmi_config(self):
        """Set IPMI configuration for each node."""
        all_machines = {}
        all_machines.update(LENOVO_LAB)
        all_machines.update(GEN9_SYSTEMS)
        all_machines.update(ARM_LAB)
        all_machines.update(PPC_SYSTEMS)
        all_machines.update(GIGABYTE_SYSTEMS)
        for mac in all_machines.keys():
            # run maas-cli command to search node by mac and return system_id
            out, _ = self._run_maas_cli(
                ["machines", "read", "mac_address=%s" % mac])
            for machine in loads(out):
                power_driver = ''
                if mac in ARM_LAB:
                    power_user = power_pass = 'admin'
                elif mac in PPC_SYSTEMS:
                    power_user = ''
                    power_pass = 'admin'
                    power_driver = 'LAN_2_0'
                elif mac in GIGABYTE_SYSTEMS:
                    power_user = GIGABYTE_USER
                    power_pass = GIGABYTE_PASS
                    power_driver = 'LAN_2_0'
                elif mac in GEN9_SYSTEMS:
                    power_user = GEN9_USER
                    power_pass = GEN9_PASS
                else:
                    power_user = POWER_USER
                    power_pass = POWER_PASS
                self._run_maas_cli([
                    "machine", "update", machine['system_id'],
                    "power_type=ipmi",
                    "power_parameters_power_address=" + all_machines[mac],
                    "power_parameters_power_user=" + power_user,
                    "power_parameters_power_pass=" + power_pass,
                    "power_parameters_power_driver=" + power_driver,
                ])

    def test_start_commissioning_machines(self):
        # Use maas-cli to accept all nodes.
        output, _ = self._run_maas_cli(["machines", "accept-all"])
        for node in loads(output):
            self.assertEqual(node['status'], 1)

    @timeout(10 * 60)
    def test_check_nodes_ready(self):
        self._wait_machines(NODE_STATUS.READY)

    def _assert_node_results_valid(self, result_type, nodes, results):
        self.assertIn(
            result_type, (RESULT_TYPE.COMMISSIONING, RESULT_TYPE.INSTALLATION)
        )
        system_ids = [m['system_id'] for m in nodes]
        results_system_ids = set(
            [r['node']['system_id'] for r in results]
        )
        self.assertThat(results_system_ids, ContainsAll(system_ids))
        for result in results:
            self.assertThat(result, ContainsDict({
                "data": Not(Is(None)),
                "result_type": Equals(result_type),
                "script_result": Equals(0),
            }))

    def test_read_commissioning_results(self):
        # Get details about the commissioned machines so we can check
        # things about the commissioning results.
        ready_machines = get_machines_with_status(self, NODE_STATUS.READY)
        # Get all commissioning results
        output, _ = self._run_maas_cli([
            "node-results", "read",
            "result_type={}".format(RESULT_TYPE.COMMISSIONING)
        ])
        commissioning_results = loads(output)
        self._assert_node_results_valid(
            RESULT_TYPE.COMMISSIONING,
            ready_machines, commissioning_results,
        )

    def test_apply_tag_to_all_machines(self):
        # Use maas-cli to set a tag on all nodes.
        output, _ = self._run_maas_cli(
            ["tags", "create", "name=all", "definition=true()",
             "comment=A tag present on all nodes"])
        tag = loads(output)
        self.assertEqual(tag['name'], "all")

    @timeout(10 * 60)
    def test_check_tag_applied_to_all_machines(self):
        # Wait for all nodes to have new tag applied.
        expected_node_count = self.get_node_count()
        while True:
            output, _ = self._run_maas_cli(["tag", "machines", "all"])
            nodes = loads(output)
            if len(nodes) == expected_node_count:
                break
            sleep(5)

    @skipUnless(TEST_JUJU, "Not testing juju")
    def test_configure_juju(self):
        # Proxy is currently broken: disable using the lab's proxy
        # as parent for now.
        token_str = get_token_str()
        # Workaround bug 972829 (in juju precise).
        server_url = MAAS_URL.replace('/MAAS', ':80/MAAS')
        with tempfile.NamedTemporaryFile() as cloud_yaml:
            # Create cloud configuration
            setup_juju_cloud(
                server=server_url, cloud_yaml=cloud_yaml
            )
            # Add cloud to Juju
            retcode, _, _ = self._run_juju_command(
                ['add-cloud', 'testmaas', cloud_yaml.name]
            )
            self.assertThat(retcode, Equals(0))
        retcode, list_clouds_output, _ = self._run_juju_command(
            ['list-clouds']
        )
        self.assertThat(retcode, Equals(0))
        self.assertThat(list_clouds_output, Contains("testmaas"))
        # Setup credentials
        credentials_path = os.path.expanduser(
            '~/.local/share/juju/credentials.yaml'
        )
        with open(credentials_path, mode='w', encoding='utf8') as credentials:
            setup_juju_authentication(
                oauth=token_str, credentials_yaml=credentials
            )
        retcode, list_creds_output, _ = self._run_juju_command(
            ['list-credentials']
        )
        self.assertThat(retcode, Equals(0))
        self.assertThat(list_creds_output, Contains("testmaas"))
        setup_local_dns()

    @timeout(60 * 60)
    @skipUnless(TEST_JUJU, "Not testing juju")
    def test_juju_bootstrap(self):
        # Wait a bit to let all the nodes go down.
        # XXX: rvb 2013-04-23 bug=1171418
        sleep(30)
        # Proxy has to be set while bootstrapping
        with tempfile.NamedTemporaryFile() as bootstrap_config:
            bootstrap_config.write("""
http-proxy: {proxy}
https-proxy: {proxy}""".format(proxy=HTTP_PROXY).encode('utf8'))
            bootstrap_config.file.flush()
            env = os.environ.copy()
            env['http_proxy'] = HTTP_PROXY
            env['https_proxy'] = HTTP_PROXY
            retcode, _, _ = self._run_juju_command([
                'bootstrap', 'testmaas', 'autopkgtest',
                '--config={config}'.format(config=bootstrap_config.name)
            ], env=env)
            self.assertThat(retcode, Equals(0))

    @skipUnless(TEST_JUJU, "Not testing juju")
    def test_juju_deploy_postgresql(self):
        # Deploy postgresql.
        env = os.environ.copy()
        env['http_proxy'] = HTTP_PROXY
        env['https_proxy'] = HTTP_PROXY
        retcode, _, _ = self._run_juju_command(
            ["deploy", "postgresql"], env=env
        )
        self.assertThat(retcode, Equals(0))
        self._wait_machines_running(1)
        self._wait_units_started('postgresql', 1)

    @skipUnless(TEST_JUJU, "Not testing juju")
    def test_juju_deploy_ubuntu_container(self):
        # Deploy ubuntu charm.
        env = os.environ.copy()
        env['http_proxy'] = HTTP_PROXY
        env['https_proxy'] = HTTP_PROXY
        retcode, _, _ = self._run_juju_command(
            ["deploy", "ubuntu", "--to", "lxd:0"], env=env
        )
        self.assertThat(retcode, Equals(0))
        # Wait for ubuntu charm to finish deploying
        self._wait_machines_running(1)
        units = self._wait_units_started('ubuntu', 1)
        public_address = units[0]['public-address']
        # Get the container info from MAAS
        output, _ = self._run_maas_cli(['devices', 'read'])
        containers = [
            device for device in loads(output) if device['parent'] is not None
        ]
        self.assertThat(containers, HasLength(1))
        # Verify that the IP address which matches the juju one is static
        ip_address = None
        for interface in containers[0]['interface_set']:
            for link in interface['links']:
                if link['ip_address'] == public_address:
                    self.assertThat(link['mode'], Equals('static'))
                    ip_address = link['ip_address']
        dynamic_range = IPRange(
            REGION_DHCP_CONFIG['ip_range_low'],
            REGION_DHCP_CONFIG['ip_range_high'],
        )
        self.assertThat(dynamic_range, Not(Contains(ip_address)))
        self.assertThat(public_address, Equals(ip_address))
        # Verify that we can SSH to the container
        ssh_command = [
            'ssh', 'ubuntu@{ip}'.format(ip=ip_address),
            'echo', 'test', '>', 'test;', 'cat', 'test'
        ]
        _, stdout, _ = run_command(ssh_command)
        self.assertThat(stdout.strip(), Equals('test'))

    nb_deployed_machines = 2

    def _wait_all_deployed(self):
        if TEST_JUJU:
            return self._wait_machines(
                NODE_STATUS.DEPLOYED,
                num_expected=self.nb_deployed_machines,
            )
        else:
            return self._wait_machines(NODE_STATUS.DEPLOYED)

    @timeout(5*60)
    @skipIf(TEST_JUJU, "Tested allocation/deployment with Juju.")
    def test_allocate_machines(self):
        # Get all nodes that are Ready
        ready_machines = self._wait_machines(NODE_STATUS.READY)
        for machine in ready_machines:
            system_id = machine['system_id']
            self._run_maas_cli(
                ['machines', 'allocate', 'system_id=' + system_id]
            )
        self._wait_machines(
            NODE_STATUS.ALLOCATED,
        )

    def _deploy_alternative_os(self, osystem):
        allocated_machines = get_machines_with_status(
            self,
            NODE_STATUS.ALLOCATED,
        )
        deploy_machine = None
        for machine in allocated_machines:
            if machine['architecture'] == 'amd64/generic':
                deploy_machine = machine
                break
        self.assertIsNotNone(
            deploy_machine,
            "Could not find AMD64 machine to deploy {} on".format(osystem),
        )
        self._run_maas_cli([
            'machine', 'deploy', deploy_machine['system_id'],
            'distro_series=' + osystem,
        ])

    @timeout(10*60)
    @skipIf(TEST_JUJU, "Tested allocation/deployment with Juju.")
    def test_deploy_machine_with_centos(self):
        self._deploy_alternative_os('centos70')

    @timeout(10*60)
    @skipIf(TEST_JUJU, "Tested allocation/deployment with Juju.")
    @skipUnless(TEST_WINDOWS, "Not testing Windows")
    def test_deploy_machine_with_windows(self):
        self._deploy_alternative_os('win2012hvr2')

    @timeout(10*60)
    @skipIf(TEST_JUJU, "Tested allocation/deployment with Juju.")
    def test_deploy_machines(self):
        # Get all nodes that are Allocated
        allocated_machines = get_machines_with_status(
            self,
            NODE_STATUS.ALLOCATED,
        )
        for machine in allocated_machines:
            system_id = machine['system_id']
            self._run_maas_cli(['machine', 'deploy', system_id])

    def _assert_expected_osystem_deployed(self, machines):
        deployed = [machine['osystem'] for machine in machines]
        expected = [get_config_value(self, 'default_osystem')]
        if not TEST_JUJU:
            expected.append('centos')
        if TEST_WINDOWS and not TEST_JUJU:
            expected.append('windows')
        self.assertThat(deployed, ContainsAll(expected))

    def _assert_expected_distro_series_deployed(self, machines):
        deployed = [machine['distro_series'] for machine in machines]
        expected = [get_config_value(self, 'default_distro_series')]
        if not TEST_JUJU:
            expected.append('centos70')
        if TEST_WINDOWS and not TEST_JUJU:
            expected.append('win2012hvr2')
        self.assertThat(deployed, ContainsAll(expected))

    @timeout(30*60)
    def test_machines_deployed(self):
        deployed_machines = self._wait_all_deployed()
        for machine in deployed_machines:
            self.assertThat(machine.get('status_name', ''), Equals('Deployed'))
        # Check that machines with all the expected releases were deployed
        self._assert_expected_osystem_deployed(deployed_machines)
        # Check that machines with all the expected series were deployed
        self._assert_expected_distro_series_deployed(deployed_machines)

    def test_read_installation_results(self):
        # Get details about the installed machines so we can check
        # things about the installation results.
        deployed_machines = get_machines_with_status(
            self,
            NODE_STATUS.DEPLOYED
        )
        # Get all installation results
        pause_until_released("installation results")
        output, _ = self._run_maas_cli([
            "node-results", "read",
            "result_type={}".format(RESULT_TYPE.INSTALLATION)
        ])
        installation_results = loads(output)
        self._assert_node_results_valid(
            RESULT_TYPE.INSTALLATION,
            deployed_machines, installation_results,
        )

    @timeout(10*60)
    def test_ip_addresses_not_in_dynamic_range(self):
        # Make sure the deployed machines have addresses in the dynamic range.
        deployed_machines = self._wait_all_deployed()
        for deployed_machine in deployed_machines:
            ips = [IPAddress(ip) for ip in deployed_machine['ip_addresses']]
            ip_range = IPRange(
                REGION_DHCP_CONFIG['ip_range_low'],
                REGION_DHCP_CONFIG['ip_range_high'])
            for ip in ips:
                self.assertThat(ip_range, Not(Contains(ip)))

    @timeout(10*60)
    def test_ip_addresses_not_in_bmc_range(self):
        # Make sure that the deployed machines do not have addresses
        # in the range reserved for the BMCs
        deployed_machines = self._wait_all_deployed()
        for deployed_machine in deployed_machines:
            ips = [IPAddress(ip) for ip in deployed_machine['ip_addresses']]
            ip_range = IPRange(BMC_START_IP, BMC_END_IP)
            for ip in ips:
                self.assertThat(ip_range, Not(Contains(ip)))

    def test_ssh_to_machines(self):
        # For each deployed machine, attempt to ssh to it and run a
        # simple command
        deployed_machines = get_machines_with_status(
            self,
            NODE_STATUS.DEPLOYED
        )
        for machine in deployed_machines:
            if machine['osystem'] == 'windows':
                continue
            ip = machine['ip_addresses'][0]
            user = machine['osystem']  # ssh user is the same as osystem
            ssh_command = [
                'ssh', '{user}@{ip}'.format(user=user, ip=ip),
                'echo', 'test', '>', 'test;', 'cat', 'test'
            ]
            _, stdout, _ = run_command(ssh_command)
            self.assertThat(stdout.strip(), Equals('test'))

    def test_ssh_add_apt_repository_and_install(self):
        deployed_machines = get_machines_with_status(
            self,
            NODE_STATUS.DEPLOYED
        )
        # Make sure we do this only on machines deployed with Ubuntu
        deployed_machines = [
            machine for machine in deployed_machines
            if machine['osystem'] == 'ubuntu'
        ]
        machine_ip = deployed_machines[0]['ip_addresses'][0]
        ssh_proxy_sudo = [
            'ssh', '-t', 'ubuntu@{ip}'.format(ip=machine_ip),
            'http_proxy={proxy}'.format(proxy=HTTP_PROXY),
            'https_proxy={proxy}'.format(proxy=HTTP_PROXY),
            'sudo', '-E',
        ]
        ret, _, _ = run_command(
            ssh_proxy_sudo + ['add-apt-repository', 'ppa:maas/next', '-y']
        )
        self.assertThat(ret, Equals(0))
        ret, _, _ = run_command(
            ssh_proxy_sudo + ['apt', 'update', '-y']
        )
        self.assertThat(ret, Equals(0))
        ret, _, _ = run_command(
            ssh_proxy_sudo + ['apt', 'install', 'curtin', '-y']
        )
        self.assertThat(ret, Equals(0))

    @timeout(30*60)
    def test_enter_rescue_mode(self):
        deployed_machines = get_machines_with_status(
            self,
            NODE_STATUS.DEPLOYED
        )
        deployed_machines = [
            machine for machine in deployed_machines
            if machine['osystem'] == 'ubuntu'
        ]
        machine_id = deployed_machines[0]['system_id']
        _, _ = self._run_maas_cli(['machine', 'rescue-mode', machine_id])
        rescue_mode_machines = self._wait_machines(
            NODE_STATUS.RESCUE_MODE, num_expected=1
        )
        # Given that the machine is now in rescue mode, $HOME should be empty
        machine_ip = rescue_mode_machines[0]['ip_addresses'][0]
        ls_command = ['ssh', 'ubuntu@{ip}'.format(ip=machine_ip), 'ls']
        _, stdout, _ = run_command(ls_command)
        self.assertThat(stdout, Equals(''))
        cmdline_command = [
            'ssh', 'ubuntu@{ip}'.format(ip=machine_ip),
            'cat', '/proc/cmdline'
        ]
        _, stdout, _ = run_command(cmdline_command)
        self.assertThat(stdout, Contains('iscsi_target_name'))

    @timeout(30*60)
    def test_exit_rescue_mode(self):
        rescue_mode_machines = get_machines_with_status(
            self,
            NODE_STATUS.RESCUE_MODE
        )
        machine_id = rescue_mode_machines[0]['system_id']
        _, _ = self._run_maas_cli(['machine', 'exit-rescue-mode', machine_id])
        # Wait until all machines are deployed again
        deployed_machines = self._wait_all_deployed()
        # Given that the machine is now back to deployed, the test file we
        # added should still be present
        ex_rescue_mode_machines = [
            machine for machine in deployed_machines
            if machine['system_id'] == machine_id
        ]
        self.assertThat(ex_rescue_mode_machines, HasLength(1))
        ex_rescue_mode_machine = ex_rescue_mode_machines[0]
        machine_ip = ex_rescue_mode_machine['ip_addresses'][0]
        ssh_command = [
            'ssh', 'ubuntu@{ip}'.format(ip=machine_ip), 'cat', 'test'
        ]
        # Keep trying to ssh until we get a succesful result, at which point
        # we can check if the state of the machine was preserved. If we never
        # do then we'll time out per the @timeout decorator
        while True:
            ret, stdout, _ = run_command(ssh_command)
            if ret == 0:
                break  #
            sleep(30)
        self.assertThat(stdout.strip(), Equals('test'))
        cmdline_command = [
            'ssh', 'ubuntu@{ip}'.format(ip=machine_ip),
            'cat', '/proc/cmdline'
        ]
        _, stdout, _ = run_command(cmdline_command)
        self.assertThat(stdout, Not(Contains('iscsi_target_name')))

    def test_release_machines(self):
        deployed_machines = get_machines_with_status(
            self,
            NODE_STATUS.DEPLOYED
        )
        for machine in deployed_machines:
            system_id = machine['system_id']
            self._run_maas_cli(['machine', 'release', system_id])

    @timeout(30 * 60)
    def test_machines_released_and_ready(self):
        released_machines = self._wait_machines(
            NODE_STATUS.READY
        )
        for machine in released_machines:
            self.assertThat(machine.get('status_name', ''), Equals('Ready'))

    @classmethod
    def dump_database(cls):
        """Dump the Django DB to /var/log/maas."""
        _, stdout, stderr = run_command([
            "sudo", maas_region_admin, "dumpdata",
            "--indent", "4",
            "--exclude", "maasserver.candidatename",
        ])
        log_dir = "/var/log/maas"
        stdout_path = os.path.join(log_dir, 'dumpdata.stdout')
        with open(stdout_path, 'wb') as w_file:
            w_file.write(stdout.encode('utf-8'))
        stderr_path = os.path.join(log_dir, 'dumpdata.stderr')
        with open(stderr_path, 'wb') as w_file:
            w_file.write(stderr.encode('utf-8'))

    @classmethod
    def copy_juju_log(cls):
        # Use timeout because juju debug-log never returns (known juju bug).
        _, stdout, _ = run_command(
            ["timeout", "10", "juju", "debug-log", "--replay", "-l", "TRACE"]
        )
        log_dir = "/var/log/maas"
        stdout_path = os.path.join(log_dir, 'juju-log-replay.stdout')
        with open(stdout_path, 'wb') as w_file:
            w_file.write(stdout.encode('utf-8'))
        run_command(
            ["juju", "scp", "0:/var/log/juju/*", log_dir]
        )

    @classmethod
    def tearDownClass(cls):
        cls.dump_database()