~andreserl/maas/qa-lab-tests-machines

from __future__ import (
    absolute_import,
    print_function,
    unicode_literals,
    )

str = None

__metaclass__ = type
__all__ = []

import errno
import os
import pipes
import platform
import re
import sys
from contextlib import closing
from distutils.version import StrictVersion
from itertools import count
from shutil import copytree
from textwrap import dedent
from time import sleep
from unittest import (
    skipIf,
    skipUnless,
    SkipTest,
    TestLoader,
)
try:
    from urllib2 import (
        urlopen,
        HTTPError,
    )
except ImportError:
    from urllib.request import urlopen
    from urllib.error import HTTPError

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

from netaddr import (
    IPAddress,
    IPRange,
)
from simplejson import loads
from testtools import TestCase
from testtools.content import (
    content_from_file,
    content_from_stream,
    text_content,
)
from testtools.matchers import (
    Contains,
    ContainsAll,
)
from timeout import timeout
# Import `pause_until_released` if you need to block execution at a given state
# so you can examine the machines in the testing pool.
from utils import (
    assertCommandReturnCode,
    assertStartedUpstartService,
    change_logs_permissions,
    filter_dict,
    get_maas_revision,
    get_maas_version,
    get_ubuntu_version,
    is_juju_core,
    retries,
    run_command,
    pause_until_released  # If this is not used below, linters will complain.
)
import yaml
import zmq


# Needed by the imports below - linters will complain.
sys.path.insert(0, "/usr/share/maas")
os.environ['DJANGO_SETTINGS_MODULE'] = 'maas.settings'

import django
django.setup()

from django.core.exceptions import AppRegistryNotReady
from django.core.management import call_command
from django.contrib.auth.models import User

from maasserver.enum import NODE_STATUS
from maasserver.models.user import get_creds_tuple
from maasserver.utils import get_local_cluster_UUID
from apiclient.creds import convert_tuple_to_string

from config import (
    ADMIN_USER,
    ARM_LAB,
    CLUSTER_CONTROLLER_IP,
    DO_NOT_TEST_JUJU,
    DO_NOT_USE_ARM_NODES,
    HTTP_PROXY,
    JUJU_CONFIG,
    LAB_DNS_CONFIG,
    LENOVO_LAB,
    MAAS_URL,
    NODE_SERIES,
    PASSWORD,
    POWER_PASS,
    POWER_USER,
    SQUID_DEB_PROXY_URL,
    USER_DATA_URL,
    USE_ARM_NODES,
    USE_CC_NODES,
)


def make_juju_config():
    config = JUJU_CONFIG
    if not is_juju_core():
        config = "%s%s" % (config, "    juju-origin: ppa\n")
    return config


def setup_juju_config(server, oauth, series):
    juju_dir = os.path.expanduser('~/.juju')
    try:
        os.mkdir(juju_dir)
    except OSError:
        pass
    config_content = make_juju_config() % (server, oauth, series)
    juju_configfile = os.path.join(juju_dir, 'environments.yaml')
    juju_configfile_fd = open(juju_configfile, 'w')
    juju_configfile_fd.write(config_content.encode('utf-8'))


def get_token_str():
    admin = User.objects.get(username=ADMIN_USER)
    token = admin.tokens.all()[0]
    return convert_tuple_to_string(get_creds_tuple(token))


def setup_ssh():
    home_dir = os.path.expanduser('~/')
    ssh_dir = os.path.join(home_dir, '.ssh')
    # Setup ssh keys.
    ssh_key = os.path.join(ssh_dir, 'id_rsa')
    if not os.path.exists(ssh_key):
        try:
            os.mkdir(ssh_dir)
        except OSError:
            pass
        run_command(['ssh-keygen', '-t', 'rsa', '-N', '', '-f', ssh_key])
    # Setup ssh config.
    ssh_config = os.path.join(ssh_dir, 'config')
    with open(ssh_config, 'w') as f:
        f.write('StrictHostKeyChecking no')


def setup_local_dns():
    content = open('/etc/resolv.conf').read().decode('ascii')
    content = 'nameserver 127.0.0.1\n' + content
    with open('/etc/resolv.conf', 'w') as f:
        f.write(content.encode('utf-8'))


DEB_PROXY_CONFIG = """
cache_peer %s parent %s 0 no-query no-digest
never_direct allow all
""" % (urlparse(SQUID_DEB_PROXY_URL).hostname,
       urlparse(SQUID_DEB_PROXY_URL).port)


# Configure the region proxy to use the proxy.
def setup_deb_proxy():
    config = '/usr/share/maas/maas-proxy.conf'
    content = open(config).read().decode('ascii')
    content = content + DEB_PROXY_CONFIG
    with open(config, 'w') as f:
        f.write(content.encode('utf-8'))
    run_command(['sudo', 'service', 'maas-proxy', 'restart'])

tests_order = [
    "test_create_admin",
    "test_restart_dbus_avahi",
    "test_update_maas_url",
    "test_restart_provisioning_server_systemctl",
    "test_check_initial_services_systemctl",
    "test_check_rpc_info",
    "test_update_preseed_arm",
    "test_login_api",
    "test_maas_logged_in",
    "test_set_http_proxy",
    # From this point on, we can connect to the external world.
    "test_import_boot_resources", # Start importing now, check later.
    "test_rack_connected",
    "test_create_dynamic_range",
    "test_set_up_dhcp_vlan",
    "test_check_dhcp_service_systemctl",
    "test_update_dns_config_systemctl",
    "test_add_new_zones",
    "test_list_zones",
    "test_delete_zone",
    "test_add_new_spaces",
    "test_add_subnet_to_space",
    "test_list_spaces",
    "test_list_subnets",
    "test_delete_subnet",
    "test_delete_space",
    "test_add_new_fabrics",
    "test_add_vlan_to_fabric",
    "test_list_fabrics",
    "test_list_vlans",
    "test_delete_vlan",
    "test_delete_fabric",
    "test_imported_boot_resources_region", # Check the images imported before.
    "test_imported_boot_resources_rack", # Check rack imported images
    # From this point on, we can enlist and deploy nodes.
    "test_boot_nodes_enlist",
    "test_add_ssh_key",  # This is needed to deploy nodes.
    "test_check_machines_declared",
    "test_assign_machines_to_test_zone",
    "test_set_machines_ipmi_config",
    "test_commission_nodes",
    "test_check_nodes_ready",
    # At this point, we can start working with Juju.
    "test_configure_juju",
    "test_juju_bootstrap",
    "test_apply_tag_to_all_machines",
    "test_juju_bootstrapped",
    "test_check_tag_applied_to_all_machines",
    "test_juju_deploy_mysql",
    "test_machines_deployed",
    "test_static_ip_addresses",
]

def sorting_method(ignored, first_test, second_test):
    return tests_order.index(first_test) - tests_order.index(second_test)


TestLoader.sortTestMethodsUsing = sorting_method

# The 'maas-cli' has been renamed to 'maas' in version 1931.
maascli = 'maas' if get_maas_revision() >= 1931 else 'maas-cli'

# The 'maas-region-admin' has been renamed to 'maas-region' in version .
maas_region_admin = (
    'maas-region' if get_maas_revision() >= 4738 else 'maas-region-admin')

# Uses the pre-1.6 API, where a cluster interface had no "name" field.
# MAAS will take the name from the "interface" field, as long as it's
# a unique name within the cluster.  Newer clients would pass a name.
REGION_DHCP_CONFIG = {
    "ip": "10.245.136.6",
    "interface": "eth1",
    "subnet_mask": "255.255.248.0",
    "broadcast_ip": "10.245.143.255",
    "router_ip": "10.245.136.1",
    "management": 2,
    "ip_range_low": "10.245.136.10",
    "ip_range_high": "10.245.136.200",
    # TODO: static ranges no longer exist
    "static_ip_range_low": "10.245.136.201",
    "static_ip_range_high": "10.245.136.250",
    # add cidr as it is now default in new networking configs.
    "cidr": "10.245.136.0/21"
}


class TestMAASIntegration(TestCase):

    def get_node_count(self):
        """The number of available nodes."""
        count = len(LENOVO_LAB)
        if USE_ARM_NODES:
            count += len(ARM_LAB)
        return count

    def _run_maas_cli(self, args):
        for _ in range(10):
            retcode, output, err = run_command([maascli, "maas"] + args)
            if "503 - MAAS is starting" in output:
                sleep(30)  # If MAAS is starting, wait 30 seconds and retry.
            else:
                break

        self.addDetail(
            'retcode for %s maas %s' % (maascli, args),
            text_content('%d' % retcode))
        self.addDetail(
            'stdout for %s maas %s' % (maascli, args),
            text_content(output))
        self.addDetail(
            'stderr for %s maas %s' % (maascli, args),
            text_content(err))
        return (output, err)

    def _update_dhcpd_apparmor_profile(self):
        """Workaround for raring due to bug 1107686."""
        with open("/etc/apparmor.d/usr.sbin.dhcpd", "r+") as dhcpd_fd:
            dhcpd_file = dhcpd_fd.read().decode('ascii')
            dhcpd_file = dhcpd_file.replace(
                'network packet packet,',
                'network packet packet,\n  network packet raw,')
            dhcpd_fd.seek(0)
            dhcpd_fd.write(dhcpd_file.encode('utf-8'))
        cmd = ["service", "apparmor", "reload"]
        expected_output = '* Reloading AppArmor profiles'
        assertCommandReturnCode(self, cmd, expected_output)

    def _set_up_dhcp(self, primary_rack_system_id, dhcp_config):
        # Support the two following cases:
        #  1. eth1 is auto-detected
        #  2. eth1  not auto-dectected. in 2.0 this would be a bug.
        rack, err = self._run_maas_cli(
            ['rack-controller', 'read', primary_rack_system_id])
        rack_controller = loads(rack)

        subnet_list, err = self._run_maas_cli(
            ['subnets', 'read'])
        subnets = loads(subnet_list)

        for interface in rack_controller['interface_set']:
            for link in interface['links']:
                if dhcp_config['ip'] == link['ip_address']:
                    rack_interface_link = link
                    break

        rack_fabric = rack_interface_link['subnet']['vlan']['fabric']
        rack_vlan = rack_interface_link['subnet']['vlan']['name']

        maas_dhcp_cmd = [
             'vlan', 'update', rack_fabric, rack_vlan, 'dhcp_on=True',
             'primary_rack=%s' % primary_rack_system_id]

        output, err = self._run_maas_cli(maas_dhcp_cmd)

        try:
            vlan_result = loads(output)
        except:
            pass
            #from maas.provisioningserver.service_monitor import get_service_state
            pause_until_released(ng_list.__repr__())
        # The JSON object returned by MAAS doesn't include the router_ip
        # address, so let's remove it from the dhcp_config before comparing.
        self.assertEqual(True, vlan_result['dhcp_on'])

    def _boot_nodes(self):
        # Run ipmipower to boot up nodes.
        for ipmi_address in LENOVO_LAB.values():
            self.power_off(ipmi_address, POWER_USER, POWER_PASS)
            self.power_on(ipmi_address, POWER_USER, POWER_PASS)
        if USE_ARM_NODES:
            for ipmi_address in ARM_LAB.values():
                self.power_off(ipmi_address, 'admin', 'admin')
                self.power_on(ipmi_address, 'admin', 'admin')

    def power_on(self, ip, user, password):
        cmd = ["ipmipower", "-h", ip, "-u", user, "-p", password, "--on"]
        expected_output = "%s: ok" % ip
        assertCommandReturnCode(self, cmd, expected_output)

    def power_off(self, ip, user, password):
        cmd = ["ipmipower", "-h", ip, "-u", user, "-p", password, "--off"]
        expected_output = "%s: ok" % ip
        assertCommandReturnCode(self, cmd, expected_output)

    def _get_machines_with_status(self, status):
        """Retrieve from the API all nodes with `status` (or substatus)."""
        # Start by listing all nodes.  The output will be shown on failure, so
        # even if we can filter by status, doing it client-side may provide
        # more helpful debug output.
        output, err = self._run_maas_cli(["machines", "read"])
        machine_list = loads(output)
        # In 1.9 we used to look at the substatus for new status's. However 2.0+
        # we simply look at the status
        machines = [
            machine for machine in machine_list
            if status == machine['status']
        ]
        return machines

    def _wait_machines(self, status=None, num_expected=None):
        """Wait for `num_expected` nodes with status `status`."""
        if num_expected is None:
            num_expected = self.get_node_count()
        self.addDetail(
            'wait_%d_%d' % (num_expected, status),
            text_content(
                "Waiting for %d node(s) with status %d."
                % (num_expected, status)))

        filtered_list = self._get_machines_with_status(status)
        while len(filtered_list) != num_expected:
            sleep(5)
            filtered_list = self._get_machines_with_status(status)
        return filtered_list

    def _run_juju_command(self, args, env=None):
        command = ["juju", "--debug"]
        command.extend(args)
        retcode, output, err = run_command(command, env=env)
        command_name = " ".join(map(pipes.quote, command))
        self.addDetail(command_name, text_content(
            output.decode("UTF-8", "replace")))
        self.addDetail(command_name, text_content(
            err.decode("UTF-8", "replace")))
        return output

    def get_juju_status(self):
        status_output = self._run_juju_command(["status"])
        status = yaml.safe_load(status_output)
        return status

    def _wait_machines_running(self, nb_machines):
        """Wait until at least `nb_machines` have their agent running."""
        while True:
            status = self.get_juju_status()
            if status is not None:
                machines = status['machines'].values()
                running_machines = [
                    machine for machine in machines
                    if machine.get('agent-state', '') in [
                        'running', 'started']]
                if len(running_machines) >= nb_machines:
                    break
            sleep(20)

    def _wait_units_started(self, service, nb_units):
        """Wait until a service has at least `nb_units` units."""
        while True:
            status = self.get_juju_status()
            try:
                units = status['services'][service]['units'].values()
            except KeyError:
                units = []
            started_units = [
                unit for unit in units
                if unit.get('agent-state', '') == 'started']
            if len(started_units) >= nb_units:
                break
            sleep(20)

    def test_create_admin(self):
        """Run maas createsuperuser."""
        cmd_output = call_command(
            "createadmin", username=ADMIN_USER, password=PASSWORD,
            email="example@canonical.com", noinput=True)
        self.assertEqual(cmd_output, None)

    # Since revision 1828, MAAS doesn't use avahi/dbus anymore so
    # we do not need to restart these daemons.
    @skipIf(
        get_maas_revision() >= 1828, "Avahi/DBUS are not used anymore")
    def test_restart_dbus_avahi(self):
        cmd = ["service", "dbus", "restart"]
        expected_output = 'dbus start/running'
        assertCommandReturnCode(self, cmd, expected_output)
        cmd = ["service", "avahi-daemon", "restart"]
        expected_output = 'avahi-daemon start/running'
        assertCommandReturnCode(self, cmd, expected_output)

    @skipIf(
        os.path.exists("/etc/maas/maas_local_settings.py"),
        "local_config_set is not available")
    def test_update_maas_url(self):
        # XXX: allenap: Using a debconf file to set the initial value of the
        # MAAS URL should not work; see test_update_maas_url_old(), but I'm
        # not sure if that's true.
        cmd = [
            "maas-region", "local_config_set", "--maas-url",
            "http://10.245.136.6/MAAS",
        ]
        assertCommandReturnCode(self, cmd, "")
        # Restart apache2.
        cmd = ["service", "apache2", "restart"]
        for retry in retries(delay=10, timeout=60):
            retcode, output, err = run_command(cmd)
            if retcode == 0:
                break
        else:
            self.fail("apache2 cannot be restarted")

    @skipIf(
        StrictVersion(get_ubuntu_version()) < StrictVersion('15.10'),
        'From 15.10 onwards, we use systemd')
    def test_restart_provisioning_server_systemctl(self):
        # Restart the provisioning server as MAAS is installed before the
        # network is properly configured, which makes clusterd listen only in
        # the 10.0.2.15 interface.
        cmd = ["systemctl", "restart", "maas-rackd"]
        retcode, output, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertEqual('', output)  # No news is good news
        self.assertEqual('', err)

    @skipIf(
        StrictVersion(get_ubuntu_version()) < StrictVersion('15.10'),
        'From 15.10 onwards, we use systemd')
    def test_check_initial_services_systemctl(self):
        cmd = ["systemctl", "status", "maas-rackd"]
        retcode, output, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertIn('Active: active (running)', output)
        self.assertEqual('', err)

    def test_check_rpc_info(self):
        # Ensure that the region is publishing RPC info to the clusters. This
        # is a reasonable indication that the region is running correctly.
        url = "%s/rpc/" % MAAS_URL.rstrip("/")
        self.addDetail('url', text_content(url))
        errors = []
        attempts = count(1)
        for elapsed, remaining in retries(timeout=300, delay=10):
            attempt = next(attempts)
            try:
                with closing(urlopen(url)) as response:
                    details = content_from_stream(response, buffer_now=True)
            except HTTPError as error:
                errors.append(error)
            else:
                self.addDetail('info', details)
                break
        else:
            self.addDetail('errors', text_content(
                "\n--\n".join([error for error in errors])))
            self.fail(
                "RPC info not published after %d attempts over %d seconds."
                % (attempt, elapsed))

    @skipIf(DO_NOT_USE_ARM_NODES, "Don't test ARM nodes")
    def test_update_preseed_arm(self):
        # XXX: matsubara add workaround to boot arm nodes.
        try:
            # Try the old location first, the one used before the templates
            # were moved to /etc/maas/.
            userdata_fd = open(
                "/usr/share/maas/preseeds/enlist_userdata", "rb+")
        except IOError:
            userdata_fd = open("/etc/maas/preseeds/enlist_userdata", "rb+")
        userdata = userdata_fd.read().decode('utf-8')
        url = 'http://ports.ubuntu.com/ubuntu-ports'
        userdata += '\n' + dedent("""\
            apt_sources:
            - source: "deb %s precise-proposed main"
            """) % url
        userdata_fd.seek(0)
        userdata_fd.write(userdata.encode('utf-8'))
        userdata_fd.close()

    def test_login_api(self):
        token_str = get_token_str()
        api_url = MAAS_URL + "/api/2.0/"
        cmd = [maascli, "login", "maas", api_url, token_str]
        expected_output = "\nYou are now logged in to the MAAS server"
        for _ in range(30):
            retcode, output, err = run_command(cmd, env=os.environ.copy())
            if retcode == 0:
                break
            elif retcode == 1 and 'Expected application/json' in err:
                # The cli may error with 'Expected application/json, got:
                # text/html; charset=utf-8\n' with retcode 1 before all pieces
                # are up and running.
                sleep(10)  # We need to give MAAS some time.
        self.assertIn(expected_output, output)

    def test_maas_logged_in(self):
        """Make sure MAAS is successfully logged in. Also, it signals a handy
        place to put a pause."""
        # pause_until_released('test_maas_logged_in')
        cmd = ["maas", "list"]
        retcode, output, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertEqual('', err)

    def test_set_http_proxy(self):
        output, err = self._run_maas_cli([
            "maas", "set-config", "name=http_proxy",
            "value=%s" % HTTP_PROXY])
        self.assertThat(output, Contains("OK"))
        output, err = self._run_maas_cli([
            "maas", "get-config", "name=http_proxy"])
        self.assertThat(output, Contains(HTTP_PROXY))

        # Check that the user-data passed on to nodes have the proxy address.
        output = urlopen(USER_DATA_URL).read().decode('utf-8')
        self.assertThat(output, Contains(
            "apt_proxy: %s" % HTTP_PROXY))

    def get_master_rack(self):
        output, err = self._run_maas_cli(["rack-controllers", "list"])
        rack_controllers = loads(output)
        master_rack_controller = rack_controllers[0]
        return master_rack_controller

    # TODO: Need to re-enable this test until we have a way to check
    # rack controllers are connected, since the master UUID is gone.
    @skipIf(
        StrictVersion(get_maas_version()[:3]) > StrictVersion('1.9'),
        'Skip this test for now until we can check cluster connectivity')
    @timeout(5 * 60)
    def test_rack_connected(self):
        # The master cluster is connected and changed the uuid field of the
        # nodegroup object from 'master' to its UUID.
        name = self.get_master_rack()['uuid']
        while name == 'master':
            sleep(10)
            name = self.get_master_rack()['uuid']

    def test_create_dynamic_range(self):
        output, err = self._run_maas_cli(
            ['ipranges', 'create', 'type=dynamic',
             'start_ip=%s' % REGION_DHCP_CONFIG['ip_range_low'],
             'end_ip=%s' % REGION_DHCP_CONFIG['ip_range_high']])
        # If the range was configured correct, it should return the subnet where
        # the range belogs to. As such, we check the output to ensure it is there.
        self.assertThat(output, Contains('"cidr": "%s"' % REGION_DHCP_CONFIG['cidr']))
        # TODO: We need to read all IP ranges for the subnet and check that the range
        # has actually been created regardless whether we checked the output above.
        output, err = self._run_maas_cli(
            ['ipranges', 'read'])

    def get_rack_systemid_on_region(self, rack_controllers):
        region_rack = 4
        for rack in rack_controllers:
            if rack['node_type'] == region_rack:
                return rack['system_id']

    @timeout(10 * 60)
    def test_set_up_dhcp_vlan(self):
        output, err = self._run_maas_cli(["rack-controllers", "read"])
        rack_controllers = loads(output)
        rack_system_id = self.get_rack_systemid_on_region(rack_controllers)
        #self.assertEqual(region_rack, rack_type)
        self._set_up_dhcp(rack_system_id, REGION_DHCP_CONFIG)

        # Wait for the task to complete and create the dhcpd.conf file.
        while os.path.exists("/var/lib/maas/dhcpd.conf") is False:
            self.addDetail(
                "Waiting task create dhcpd.conf file.",
                content_from_file("/var/log/maas/maas.log"))
            sleep(2)

    @skipIf(
        StrictVersion(get_ubuntu_version()) < StrictVersion('15.10'),
        'From 15.10 onwards, we use systemd')
    # Timeout after 2 minutes, provided that it may take a while for the
    # rack controller to write dhcp config and start maas-dhcpd
    @timeout(2 * 60)
    def test_check_dhcp_service_systemctl(self):
        cmd = ["systemctl", "status", "maas-dhcpd"]
        # systemd will return 3 if a 'condition failed. This typically means that
        # /var/lib/maas/dhcpd.conf is not there yet, and we should wait for a bit
        # to see if the config is written and maas-dhcp is brought up by the rack.
        retcode, output, err = run_command(cmd)
        while retcode != 0:
            # query systemd every 3 seconds to see if maas-dhcpd us running
            sleep(3)
            retcode, output, err = run_command(cmd)
        #pause_until_released((retcode, output, err).__repr__())
        self.assertEqual(0, retcode)
        self.assertIn('Active: active (running)', output)
        self.assertEqual('', err)

    @skipIf(
        StrictVersion(get_ubuntu_version()) < StrictVersion('15.10'),
        'From 15.10 onwards, we use systemd')
    def test_update_dns_config_systemctl(self):
        dns_config = open("/etc/bind/named.conf.options", 'w')
        dns_config.write(LAB_DNS_CONFIG)
        dns_config.close()
        cmd = ["systemctl", "restart", "bind9"]
        retcode, output, err = run_command(cmd)
        self.assertEqual(0, retcode)
        self.assertEqual('', output)  # No news is good news
        self.assertEqual('', err)

    # Zones
    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_add_new_zones(self):
        # Create 2 new zones.
        output, err = self._run_maas_cli(
            ["zones", "create", "name=test-zone", "description='A test zone'"])
        zone = loads(output)
        self.assertEqual('test-zone', zone['name'])
        output, err = self._run_maas_cli(
            ["zones", "create", "name=delete-zone",
             "description='A test zone to be deleted'"])
        zone = loads(output)
        self.assertEqual('delete-zone', zone['name'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_list_zones(self):
        output, err = self._run_maas_cli(["zones", "read"])
        zones = loads(output)
        expected = [u'default', u'delete-zone', u'test-zone']
        self.assertEqual(expected, sorted([zone['name'] for zone in zones]))

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_delete_zone(self):
        _, err = self._run_maas_cli(["zone", "delete", "delete-zone"])
        # List the remaining zones after the delete command.
        output, err = self._run_maas_cli(["zones", "read"])
        zones = loads(output)
        self.assertNotIn(
            'delete-zone', sorted([zone['name'] for zone in zones]))

    # Spaces and subnets
    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Space feature only available after 1.9')
    def test_add_new_spaces(self):
        # Create 2 new spaces.
        output, err = self._run_maas_cli(
            ['spaces', 'create', 'name=test-space'])
        out_dict = loads(output)
        self.assertEqual('test-space', out_dict['name'])
        output, err = self._run_maas_cli(
            ['spaces', 'create', 'name=delete-space'])
        out_dict = loads(output)
        self.assertEqual('delete-space', out_dict['name'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Subnet feature only available after 1.9')
    def test_add_subnet_to_space(self):
        output, err = self._run_maas_cli(
            ['subnets', 'create', 'space=0', 'name=test-subnet',
             'cidr=192.168.200.0/24'])
        out_dict = loads(output)
        self.assertEqual('test-subnet', out_dict['name'])
        self.assertEqual('192.168.200.0/24', out_dict['cidr'])
        self.assertEqual('space-0', out_dict['space'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Space feature only available after 1.9')
    def test_list_spaces(self):
        output, err = self._run_maas_cli(['spaces', 'read'])
        out_dict = loads(output)
        expected = ['delete-space', 'space-0', 'test-space']
        self.assertItemsEqual(
            expected, [item['name'] for item in out_dict])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Subnet feature only available after 1.9')
    def test_list_subnets(self):
        output, err = self._run_maas_cli(['subnets', 'read'])
        out_dict = loads(output)
        self.assertEqual(2, len(out_dict), 'space-0 should now have 2 subnets')

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Subnet feature only available after 1.9')
    def test_delete_subnet(self):
        _, err = self._run_maas_cli(['subnet', 'delete', 'test-subnet'])
        output, err = self._run_maas_cli(['subnets', 'read'])
        out = loads(output)
        self.assertEqual(1, len(out), 'space-0 should now have 1 subnet')
        self.assertNotIn('test-subnet', [n['name'] for n in out])
        self.assertEqual(
            out[0]['name'], out[0]['cidr'],
            'Name and CIDR should be equal for the default subnet.')

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Space feature only available after 1.9')
    def test_delete_space(self):
        _, err = self._run_maas_cli(['space', 'delete', 'delete-space'])
        # List the remaining zones after the delete command.
        output, err = self._run_maas_cli(['spaces', 'read'])
        out_dict = loads(output)
        self.assertNotIn(
            'delete-space', [item['name'] for item in out_dict])

    # Fabrics and VLANs
    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Fabric feature only available after 1.9')
    def test_add_new_fabrics(self):
        # Create 2 new fabrics.
        output, err = self._run_maas_cli(
            ['fabrics', 'create', 'name=test-fabric'])
        out_dict = loads(output)
        self.assertEqual('test-fabric', out_dict['name'])
        output, err = self._run_maas_cli(
            ['fabrics', 'create', 'name=delete-fabric'])
        out_dict = loads(output)
        self.assertEqual('delete-fabric', out_dict['name'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'VLAN feature only available after 1.9')
    def test_add_vlan_to_fabric(self):
        output, err = self._run_maas_cli(
            ['vlans', 'create', 'test-fabric', 'name=test-vlan', 'vid=2'])
        out_dict = loads(output)
        self.assertEqual('test-vlan', out_dict['name'])
        self.assertEqual('test-fabric', out_dict['fabric'])
        self.assertEqual(2, out_dict['vid'])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'Fabric feature only available after 1.9')
    def test_list_fabrics(self):
        output, err = self._run_maas_cli(['fabrics', 'read'])
        out_dict = loads(output)
        expected = [u'fabric-0', u'fabric-1', u'delete-fabric', u'test-fabric']
        self.assertItemsEqual(
            expected, [item['name'] for item in out_dict])
        self.assertIn(
            'test-vlan',
            [[v['name'] for v in f['vlans']]
             for f in out_dict if f['name'] == 'test-fabric'][0])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'VLAN feature only available after 1.9')
    def test_list_vlans(self):
        output, err = self._run_maas_cli(['vlans', 'read', 'test-fabric'])
        out_dict = loads(output)
        expected = ['untagged', 'test-vlan']
        self.assertItemsEqual(expected, [v['name'] for v in out_dict])

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        'VLAN feature only available after 1.9')
    def test_delete_vlan(self):
        _, err = self._run_maas_cli(
            ['vlan', 'delete', 'test-fabric', 'test-vlan'])
        output, err = self._run_maas_cli(['vlans', 'read', 'test-fabric'])
        out = loads(output)
        self.assertNotIn('test-vlan', [v['name'] for v in out])
        self.assertEqual(1, len(out), 'Fabric should have only one VLAN now.')

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.9'),
        "Fabric feature only available after 1.9")
    def test_delete_fabric(self):
        _, err = self._run_maas_cli(["fabric", "delete", "delete-fabric"])
        # List the remaining zones after the delete command.
        output, err = self._run_maas_cli(["fabrics", "read"])
        out_dict = loads(output)
        self.assertNotIn(
            'delete-fabric', sorted([item['name'] for item in out_dict]))

    def test_import_boot_resources(self):
        output, err = self._run_maas_cli(
            ["boot-resources", "import"])
        expected_output = (
            "Import of boot resources started")
        self.assertThat(output, Contains(expected_output))

    @timeout(60 * 60)  # Allow for up to one hour
    def test_imported_boot_resources_region(self):
        expected_resources = set([
            (u'ubuntu/xenial', 'amd64/hwe-x'),
            ])

        complete_resources = set()
        while not expected_resources.issubset(complete_resources):
            resources_output, err = self._run_maas_cli(
                ["boot-resources", "read"])
            resources = loads(resources_output)
            for resource in resources:
                resource_id = resource['id']
                resource_name = resource['name']
                resource_arch = resource['architecture']
                output, err = self._run_maas_cli(
                    ['boot-resource', 'read', '%s' % resource_id])
                resource_data = loads(output)
                for _, resource_set in resource_data['sets'].items():
                    if resource_set['complete']:
                        complete_resources.add((resource_name, resource_arch))
            self.addDetail(
                "Waiting for region to import boot resources.",
                text_content(resources_output))
            sleep(5)

    @timeout(60 * 30)  # Allow for up to 30 mins
    def test_imported_boot_resources_rack(self):
        output, err = self._run_maas_cli(["rack-controllers", "read"])
        rack_controllers = loads(output)
        rack_system_id = self.get_rack_systemid_on_region(rack_controllers)
        while True:
            output, err = self._run_maas_cli(
                ["rack-controller", "list-boot-images", rack_system_id])
            rack_images = loads(output)
            if rack_images['status'] == 'synced':
                break
            self.addDetail(
                "Waiting for rack to import boot resources.",
                text_content(output))
            sleep(5)

    def test_add_ssh_key(self):
        """Add our ssh key to the admin account."""
        setup_ssh()
        home_dir = os.path.expanduser('~/')
        ssh_dir = os.path.join(home_dir, '.ssh')
        ssh_key = open(
            os.path.join(ssh_dir, 'id_rsa.pub')).read()
        out, err = self._run_maas_cli(["sshkeys", "create", "key=%s" % ssh_key])
        out, err = self._run_maas_cli(["sshkeys", "read"])
        keys = loads(out)
        self.assertEqual(ssh_key, keys[0]['key'])

    def test_boot_nodes_enlist(self):
        self._boot_nodes()

    # @timeout(7 * 60)
    def test_check_machines_declared(self):
        self._wait_machines(NODE_STATUS.NEW)  # 0

    @skipIf(
        StrictVersion(get_maas_version()[:3]) < StrictVersion('1.5'),
        "Zone feature only available after 1.5")
    def test_assign_machines_to_test_zone(self):
        # Set two of the declared nodes to the test-zone created earlier.
        output, err = self._run_maas_cli(["machines", "read"])
        machines = loads(output)
        for machine in machines[:2]:
            self._run_maas_cli(
                ["machine", "update", machine['system_id'], "zone=test-zone"])
        # Check nodes are in the test-zone
        output, err = self._run_maas_cli(["machines", "read", "zone=test-zone"])
        machines = loads(output)
        self.assertEqual(2, len(machines))

    def test_set_machines_ipmi_config(self):
        """Set IPMI configuration for each node."""
        all_machines = {}
        all_machines.update(LENOVO_LAB)
        all_machines.update(ARM_LAB)
        for mac in all_machines.keys():
            # run maas-cli command to search node by mac and return system_id
            out, err = self._run_maas_cli(
                ["machines", "read", "mac_address=%s" % mac])
            machine_list = loads(out)
            for machine in machine_list:
                if mac in ARM_LAB:
                    power_user = power_pass = 'admin'
                else:
                    power_user = POWER_USER
                    power_pass = POWER_PASS
                self._run_maas_cli(
                    ["machine", "update", machine['system_id'], "power_type=ipmi",
                     "power_parameters_power_address=%s" % all_machines[mac],
                     "power_parameters_power_user=%s" % power_user,
                     "power_parameters_power_pass=%s" % power_pass])

    def test_commission_nodes(self):
        # Use maas-cli to accept all nodes.
        output, err = self._run_maas_cli(["machines", "accept-all"])
        for node in loads(output):
            self.assertEqual(node['status'], 1)

    @timeout(10 * 60)
    def test_check_nodes_ready(self):
        self._wait_machines(NODE_STATUS.READY)

    def test_apply_tag_to_all_machines(self):
        # Use maas-cli to set a tag on all nodes.
        output, err = self._run_maas_cli(
            ["tags", "create", "name=all", "definition=true()",
             "comment=A tag present on all nodes"])
        tag = loads(output)
        self.assertEqual(tag['name'], "all")

    @timeout(10 * 60)
    def test_check_tag_applied_to_all_machines(self):
        # Wait for all nodes to have new tag applied.
        expected_node_count = self.get_node_count()
        while True:
            output, err = self._run_maas_cli(["tag", "machines", "all"])
            nodes = loads(output)
            if len(nodes) == expected_node_count:
                break
            sleep(5)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    def test_configure_juju(self):
        # Proxy is currently broken: disable using the lab's proxy
        # as parent for now.
        setup_deb_proxy()  # Maybe it's not broken anymore
        token_str = get_token_str()
        # Workaround bug 972829 (in juju precise).
        server_url = MAAS_URL.replace('/MAAS', ':80/MAAS')
        setup_juju_config(server_url, token_str, series=NODE_SERIES)
        setup_local_dns()

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(60 * 60)
    def test_juju_bootstrap(self):
        # Wait a bit to let all the nodes go down.
        # XXX: rvb 2013-04-23 bug=1171418
        sleep(30)
        if is_juju_core():
            command = ['bootstrap', '--upload-tools']
        else:
            command = ['bootstrap']
        self._run_juju_command(command)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(60 * 60)
    def test_juju_bootstrapped(self):
        """Wait for one machine with the Juju agent up."""
        self._wait_machines_running(1)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    def test_juju_deploy_mysql(self):
        # Deploy mysql.
        self._run_juju_command(["set-env", "http-proxy=%s" % HTTP_PROXY])
        self._run_juju_command(["set-env", "https-proxy=%s" % HTTP_PROXY])
        env = os.environ.copy()
        env['http_proxy'] = HTTP_PROXY
        env['https_proxy'] = HTTP_PROXY
        self._run_juju_command(["deploy", "mysql"], env=env)
        self._wait_machines_running(2)
        self._wait_units_started('mysql', 1)

    nb_of_deployed_machines = 2

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(10)
    def test_machines_deployed(self):
        allocated_machines = self._wait_machines(
            NODE_STATUS.DEPLOYED, num_expected=self.nb_of_deployed_machines)
        params = [
            "nodes=%s" % node['system_id']
            for node in allocated_machines
            ]
        output, err = self._run_maas_cli(
            ["nodes", "deployment-status"] + params)
        deployment_statuses = loads(output).values()
        self.assertEqual(
            ["Deployed"] * self.nb_of_deployed_machines, deployment_statuses)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(10)
    def test_static_ip_addresses(self):
        # Make sure the deployed machines have addresses in the static range.
        deployed_machines = self._wait_machines(
            NODE_STATUS.DEPLOYED, num_expected=self.nb_of_deployed_machines)
        for deployed_machine in deployed_machines:
            ips = [IPAddress(ip) for ip in deployed_machine['ip_addresses']]
            ip_range = IPRange(
                REGION_DHCP_CONFIG['static_ip_range_low'],
                REGION_DHCP_CONFIG['static_ip_range_high'])
            self.assertThat(
                ip_range,
                ContainsAll(ips),
                "The IP addresses of the deployed nodes are not in the "
                "cluster interface's static range.")

    @classmethod
    def dump_database(cls):
        """Dump the Django DB to /var/log/maas."""
        ret, stdout, stderr = run_command([
            "sudo", maas_region_admin, "dumpdata",
            "--indent", "4",
            "--exclude", "maasserver.candidatename",
        ])
        log_dir = "/var/log/maas"
        stdout_path = os.path.join(log_dir, 'dumpdata.stdout')
        with open(stdout_path, 'wb') as w_file:
            w_file.write(stdout.encode('utf-8'))
        stderr_path = os.path.join(log_dir, 'dumpdata.stderr')
        with open(stderr_path, 'wb') as w_file:
            w_file.write(stderr.encode('utf-8'))

    @classmethod
    def _collect_logs(cls, log_dirs, log_dest):
        """Collect logs and configs from the test run.

        This method copies logs and configs from the test run to a known
        location so adt/auto-package-testing can copy them out of the testbed.
        """
        for log_dir in log_dirs:
            try:
                copytree(log_dir, os.path.join(log_dest, log_dir.lstrip('/')))
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
                # Directory does not exist: ignore and carry on.

    @classmethod
    def finalize(cls):
        """Collect logs and signal to the cluster tests finished."""
        # ADT_ARTIFACTS is used by adt while /var/tmp/testresults is used by
        # auto-package-testing.
        log_path = os.environ.get('ADT_ARTIFACTS', '/var/tmp/testresults')
        log_dest = os.path.join(log_path, 'maas-logs')
        log_dirs = [
            '/var/log',
            '/var/lib/maas/dhcp',
            '/etc/maas']
        cls._collect_logs(log_dirs, log_dest)
        change_logs_permissions(log_dest)

        # Signal to the cluster that the region controller tests finished.
        if not USE_CC_NODES:
            raise SkipTest("Not testing Cluster controller")
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.connect('tcp://%s:5555' % CLUSTER_CONTROLLER_IP)
        socket.send("Region controller tests finished.")

    @classmethod
    def copy_juju_log(cls):
        # Use timeout because juju debug-log never returns (known juju bug).
        ret, stdout, stderr = run_command(
            ["timeout", "10", "juju", "debug-log", "--replay", "-l", "TRACE"])
        log_dir = "/var/log/maas"
        stdout_path = os.path.join(log_dir, 'juju-log-replay.stdout')
        with open(stdout_path, 'wb') as w_file:
            w_file.write(stdout.encode('utf-8'))
        ret, stdout, stderr = run_command([
            "juju", "scp", "0:/var/log/juju/*", log_dir])

    @classmethod
    def tearDownClass(cls):
        cls.dump_database()
        #cls.copy_juju_log()
        cls.finalize()