~matsubara/maas/bootimages-cli

import os
from shutil import copytree
from simplejson import loads
import sys
from time import sleep
from testtools import TestCase
from testtools.matchers import Contains
from testtools.content import Content, text_content
from testtools.content_type import UTF8_TEXT
import unittest
from unittest import skipIf, SkipTest
import yaml
import platform
import urllib2
from urlparse import urlparse
import pipes
import errno
import zmq

from utils import (
    assertCommandReturnCode, assertStartedUpstartService,
    change_logs_permissions, is_juju_core, run_command, timeout,
    update_pxe_config, CLUSTER_CONTROLLER_IP, SQUID_DEB_PROXY_URL,
    HTTP_PROXY, get_maas_revision,
    )

sys.path.insert(0, "/usr/share/maas")
os.environ['DJANGO_SETTINGS_MODULE'] = 'maas.settings'
from django.core.management import call_command
from django.contrib.auth.models import User

from maasserver.models.user import get_creds_tuple
from maasserver.models import BootImage
from maasserver.utils import get_local_cluster_UUID
from apiclient.creds import convert_tuple_to_string

# Environment variables that can be used to configure
# what is tested.

# Series to install on deployed nodes.
NODE_SERIES = os.environ.get('NODE_SERIES', 'precise')

# Whether or not the ARM nodes should be used
# in the test.
DO_NOT_USE_ARM_NODES = bool(
    os.environ.get('DO_NOT_USE_ARM_NODES', False))


USE_ARM_NODES = not DO_NOT_USE_ARM_NODES

# We cannot use the ARM nodes with juju-core because of
# bug 1227722.
if is_juju_core():
    USE_ARM_NODES=False

# Whether or not the juju tests should be performed.
DO_NOT_TEST_JUJU = bool(
    os.environ.get('DO_NOT_TEST_JUJU', False))

# Whether or not set the http proxy. Currently only working on raring
DO_NOT_SET_PROXY = bool("raring" not in platform.linux_distribution())

# Whether or not the cluster controller nodes in different network
# should be used.
USE_CC_NODES = bool(os.environ.get('USE_CC_NODES', False))

MAAS_URL = "http://192.168.21.5/MAAS"
USER_DATA_URL = "http://10.98.3.59/MAAS/metadata/enlist/latest/user-data"
ADMIN_USER = "admin"
PASSWORD = "test"
POWER_USER = "root"
POWER_PASS = "ubuntu"

REGION_CONTROLLER_NODES = {
    "00:E0:81:DD:D5:99" : "192.168.22.33",
    "00:E0:81:DD:D1:0B" : "192.168.22.34",
    "00:E0:81:DD:D4:11" : "192.168.22.35",
    "00:E0:81:D1:B1:47" : "192.168.22.36",
    }

CLUSTER_CONTROLLER_NODES = {
    "00:E0:81:DD:D1:1B" : "192.168.22.37",
    "00:E0:81:DD:D1:2B" : "192.168.22.38",
    "00:E0:81:DD:D1:A3" : "192.168.22.39",
    "00:E0:81:DC:38:6D" : "192.168.22.40",
    "00:E0:81:DD:D0:FF" : "192.168.22.41",
    "00:E0:81:DD:D4:F9" : "192.168.22.42"
    }


LENOVO_LAB = REGION_CONTROLLER_NODES
if USE_CC_NODES:
    LENOVO_LAB.update(CLUSTER_CONTROLLER_NODES)

ARM_LAB = {
    'fc:2f:40:d8:fb:1a': '192.168.21.50',
    'fc:2f:40:fa:2f:9a': '192.168.21.51',
    'fc:2f:40:ec:73:42': '192.168.21.52',
    'fc:2f:40:d5:87:ce': '192.168.21.53'
    }

# dnssec is now enabled by default, so disable it to avoid errors from the
# forwarder server.
# See:
# http://stackoverflow.com/questions/13342003/ns-got-insecure-response-parent-indicates-it-should-be-secure
LAB_DNS_CONFIG = """
options {
    directory "/var/cache/bind";
    forwarders {
    10.98.3.6;
    };
    allow-query-cache {any;};
    dnssec-enable no;
    dnssec-validation no;
    auth-nxdomain no;    # conform to RFC1035
};
"""

JUJU_CONFIG = """
default: maas
environments:
  maas:
    type: maas
    maas-server: '%s'
    maas-oauth: '%s'
    admin-secret: 'nothing'
    default-series: %s
"""


def make_juju_config():
    config = JUJU_CONFIG
    if not is_juju_core():
        config = "%s%s" % (config, "    juju-origin: ppa\n")
    return config


def setup_juju_config(server, oauth, series):
    juju_dir = os.path.expanduser('~/.juju')
    try:
        os.mkdir(juju_dir)
    except OSError:
        pass
    config_content = make_juju_config() % (server, oauth, series)
    juju_configfile = os.path.join(juju_dir, 'environments.yaml')
    juju_configfile_fd = open(juju_configfile, 'w')
    juju_configfile_fd.write(config_content)


def get_token_str():
    admin = User.objects.get(username=ADMIN_USER)
    token = admin.tokens.all()[0]
    return convert_tuple_to_string(get_creds_tuple(token))


def setup_ssh():
    home_dir = os.path.expanduser('~/')
    ssh_dir = os.path.join(home_dir, '.ssh')
    # Setup ssh keys.
    ssh_key = os.path.join(ssh_dir, 'id_rsa')
    if not os.path.exists(ssh_key):
        try:
            os.mkdir(ssh_dir)
        except OSError:
            pass
        run_command(['ssh-keygen', '-t', 'rsa', '-N', '', '-f', ssh_key])
    # Setup ssh config.
    ssh_config = os.path.join(ssh_dir, 'config')
    with open(ssh_config, 'w') as f:
        f.write('StrictHostKeyChecking no')


def setup_local_dns():
    content = open('/etc/resolv.conf').read()
    content = 'nameserver 127.0.0.1\n' + content
    with open('/etc/resolv.conf', 'w') as f:
        f.write(content)


DEB_PROXY_CONFIG = """
cache_peer %s parent %s 0 no-query no-digest
never_direct allow all
""" % (urlparse(SQUID_DEB_PROXY_URL).hostname,
       urlparse(SQUID_DEB_PROXY_URL).port)

# Configure the region proxy to use the proxy.
def setup_deb_proxy():
    config = '/etc/squid-deb-proxy/squid-deb-proxy.conf'
    content = open(config).read()
    content = content + DEB_PROXY_CONFIG
    with open(config, 'w') as f:
        f.write(content)
    run_command(['sudo', 'service', 'squid-deb-proxy', 'restart'])

tests_order = [
    "test_create_admin",
    "test_restart_dbus_avahi",
    "test_update_maas_url",
    "test_restart_provisioning_server",
    "test_check_initial_services",
    "test_update_pxe_config",
    "test_import_pxe_files",
    "test_update_preseed_arm",
    "test_login_api",
    "test_set_http_proxy",
    "test_cluster_connected",
    "test_set_up_dhcp_region",
    "test_check_dhcp_service",
    "test_set_up_dhcp_cluster",
    "test_update_dns_config",
    "test_boot_nodes_enlist",
    "test_check_nodes_declared",
    "test_set_nodes_ipmi_config",
    "test_commission_nodes",
    "test_check_nodes_ready",
    "test_apply_tag_to_all_nodes",
    "test_check_tag_applied_to_all_nodes",
    "test_configure_juju",
    "test_juju_bootstrap",
    "test_juju_setup",
    "test_juju_deploy_mediawiki",
    "test_mediawiki_homepage",
    "test_juju_add_unit_mediawiki"
]


def sorting_method(ignored, first_test, second_test):
    return tests_order.index(first_test) - tests_order.index(second_test)


unittest.TestLoader.sortTestMethodsUsing = sorting_method


class TestMAASIntegration(TestCase):

    def get_node_count(self):
        """The number of available nodes."""
        count = len(LENOVO_LAB)
        if USE_ARM_NODES:
            count += len(ARM_LAB)
        return count

    def _run_maas_cli(self, args):
        retcode, output, err = run_command(["maas-cli", "maas"] + args)
        self.addDetail('retcode for maas-cli maas %s' %
            str(args), text_content(str(retcode)))
        self.addDetail('stdout for maas-cli maas %s' %
            str(args), text_content(str(output)))
        self.addDetail('stderr for maas-cli maas %s' %
            str(args), text_content(str(err)))
        return (output, err)

    def test_create_admin(self):
        """Run maas createsuperuser."""
        cmd_output = call_command(
            "createadmin", username=ADMIN_USER, password=PASSWORD,
            email="example@canonical.com", noinput=True)
        # No output, means admin user was created successfully.
        self.assertEqual(cmd_output, None)

    # Since revision 1828, MAAS doesn't use avahi/dbus anymore so
    # we do not need to restart these daemons.
    @unittest.skipIf(
        get_maas_revision() >= 1828, "Avahi/DBUS are not used anymore")
    def test_restart_dbus_avahi(self):
        cmd = ["service", "dbus", "restart"]
        expected_output = 'dbus start/running'
        assertCommandReturnCode(self, cmd, expected_output)
        cmd = ["service", "avahi-daemon", "restart"]
        expected_output = 'avahi-daemon start/running'
        assertCommandReturnCode(self, cmd, expected_output)

    def test_update_maas_url(self):
        #XXX: matsubara For some reason using a debconf file to set the
        # initial value of DEFAULT_MAAS_URL is not working.
        maas_fd = open("/etc/maas/maas_local_settings.py" , "r+")
        maas_file = maas_fd.read()
        maas_file = maas_file.replace(
            'DEFAULT_MAAS_URL = "http://10.0.2.15/MAAS"',
            'DEFAULT_MAAS_URL = "http://192.168.21.5/MAAS"')
        maas_fd.seek(0)
        maas_fd.write(maas_file)
        maas_fd.close()
        cmd = ["service", "apache2", "restart"]
        expected_output = ' * Restarting web server apache2'
        assertCommandReturnCode(self, cmd, expected_output)

    def test_restart_provisioning_server(self):
        # Restart the provisioning server as MAAS is installed before the
        # network is properly configured, which makes pserv listen only in
        # the 10.0.2.15 interface.
        cmd = ["service", "maas-pserv", "restart"]
        expected_output = 'maas-pserv start/running'
        assertCommandReturnCode(self, cmd, expected_output)

    def test_check_initial_services(self):
        assertStartedUpstartService(
            self, 'maas-cluster-celery',
            '/var/log/upstart/maas-cluster-celery.log')
        assertStartedUpstartService(
            self, 'maas-region-celery',
            '/var/log/upstart/maas-region-celery.log')
        assertStartedUpstartService(self, 'maas-txlongpoll')
        assertStartedUpstartService(
            self, 'maas-pserv', '/var/log/maas/pserv.log')

    def test_update_pxe_config(self):
        update_pxe_config()

    def test_import_pxe_files(self):
        env = os.environ.copy()
        env['http_proxy'] = HTTP_PROXY
        cmd = ['maas-import-pxe-files']
        expected_output = 'Downloading to temporary location'
        assertCommandReturnCode(self, cmd, expected_output, env=env)

    def test_update_preseed_arm(self):
        #XXX: matsubara add workaround to boot arm nodes.
        try:
            # Try the old location first, the one used before the templates
            # were moved to /etc/maas/.
            userdata_fd = open("/usr/share/maas/preseeds/enlist_userdata", "rb+")
        except IOError:
            userdata_fd = open("/etc/maas/preseeds/enlist_userdata", "rb+")
        userdata = userdata_fd.read()
        userdata += '\napt_sources:\n - source: "deb http://ports.ubuntu.com/ubuntu-ports precise-proposed main"'
        userdata_fd.seek(0)
        userdata_fd.write(userdata)
        userdata_fd.close()

    def test_login_api(self):
        token_str = get_token_str()
        api_url = MAAS_URL + "/api/1.0/"
        cmd = ["maas-cli", "login", "maas", api_url, token_str]
        expected_output = "\nYou are now logged in to the MAAS server"
        assertCommandReturnCode(self, cmd, expected_output)

    @skipIf(DO_NOT_SET_PROXY, "Not testing proxy.")
    def test_set_http_proxy(self):
        output, err = self._run_maas_cli([
            "maas", "set-config", "name=http_proxy",
            "value=%s" % SQUID_DEB_PROXY_URL])
        self.assertThat(output, Contains("OK"))
        output, err = self._run_maas_cli([
            "maas", "get-config", "name=http_proxy"])
        self.assertThat(output, Contains(SQUID_DEB_PROXY_URL))

        # Check that the user-data passed on to nodes have the proxy address.
        output = urllib2.urlopen(USER_DATA_URL).read()
        self.assertThat(output, Contains(
            "apt_proxy: %s" % SQUID_DEB_PROXY_URL))

    def get_master_ng_uuid(self):
        output, err = self._run_maas_cli(["node-groups", "list"])
        node_groups = loads(output)
        master_nodegroup = node_groups[0]
        return master_nodegroup['uuid']

    @timeout(5*60)
    def test_cluster_connected(self):
        # The master cluster is connected and changed the uuid field of the
        # nodegroup object from 'master' to its UUID.
        name = self.get_master_ng_uuid()
        while name == 'master':
            sleep(10)
            name = self.get_master_ng_uuid()

    def _update_dhcpd_apparmor_profile(self):
        """Workaround for raring due to bug 1107686."""
        dhcpd_fd = open("/etc/apparmor.d/usr.sbin.dhcpd", "r+")
        dhcpd_file = dhcpd_fd.read()
        dhcpd_file = dhcpd_file.replace(
            'network packet packet,',
            'network packet packet,\n  network packet raw,')
        dhcpd_fd.seek(0)
        dhcpd_fd.write(dhcpd_file)
        dhcpd_fd.close()
        cmd = ["service", "apparmor", "reload"]
        expected_output = '* Reloading AppArmor profiles'
        assertCommandReturnCode(self, cmd, expected_output)

    def _set_up_dhcp(self, uuid, dhcp_config):
        dhcp_cli_args = [
            "%s=%s" % (key, value) for key, value in dhcp_config.items()]
        maas_dhcp_cmd = [
            "node-group-interface", "update", uuid, "eth1" ] + dhcp_cli_args
        output, err = self._run_maas_cli(maas_dhcp_cmd)
        node_group = loads(output)
        # The JSON object returned by MAAS doesn't include the router_ip
        # address, so let's remove it from the dhcp_config before comparing.
        dhcp_config.pop('router_ip')
        self.assertEquals(node_group, dhcp_config)

    @timeout(5*60)
    def test_set_up_dhcp_region(self):
        release = platform.linux_distribution()[2]
        if release in ('quantal', 'raring'):
            self._update_dhcpd_apparmor_profile()

        region_uuid = get_local_cluster_UUID()

        output, err = self._run_maas_cli(["node-groups", "list"])
        node_groups = loads(output)

        ng_uuids = [ng['uuid'] for ng in node_groups]
        self.assertIn(region_uuid, ng_uuids)

        region_dhcp_config = {
            "ip": "192.168.21.5",
            "interface": "eth1",
            "subnet_mask": "255.255.255.0",
            "broadcast_ip": "192.168.21.255",
            "router_ip": "192.168.21.1",
            "management": 2,
            "ip_range_low": "192.168.21.10",
            "ip_range_high": "192.168.21.200"}

        self._set_up_dhcp(region_uuid, region_dhcp_config)

        # Wait for the task to complete and create the dhcpd.conf file.
        while os.path.exists("/etc/maas/dhcpd.conf") is False:
            self.addDetail(
                "Waiting task create dhcpd.conf file.",
                Content(UTF8_TEXT, lambda:
                    open("/var/log/maas/celery-region.log", 'r').readlines()))
            sleep(2)

    def test_check_dhcp_service(self):
        assertStartedUpstartService(
            self, 'maas-dhcp-server',
            '/var/log/upstart/maas-dhcp-server.log')

    @skipIf(not USE_CC_NODES, "Not testing cluster controller")
    @timeout(5*60)
    def test_set_up_dhcp_cluster(self):
        output, err = self._run_maas_cli(["node-groups", "list"])
        node_groups = loads(output)
        # Wait until we have two node_group since the cluster controller
        # might take some time to finish its configuration and contact the
        # region controller.
        while not len(node_groups) == 2:
            output, err = self._run_maas_cli(["node-groups", "list"])
            node_groups = loads(output)

        for ng in node_groups:
            if ng['uuid'] != get_local_cluster_UUID():
                cluster_uuid = ng['uuid']
                break
        # Configure the cluster controller DHCP server.
        cluster_dhcp_config = {
            "ip": "192.168.20.5",
            "interface": "eth1",
            "subnet_mask": "255.255.255.0",
            "broadcast_ip": "192.168.20.255",
            "router_ip": "192.168.20.1",
            "management": 2,
            "ip_range_low": "192.168.20.10",
            "ip_range_high": "192.168.20.30",
            }
        self._set_up_dhcp(cluster_uuid, cluster_dhcp_config)

        # Accept the cluster controller into the MAAS server.
        output, err = self._run_maas_cli([
            "node-groups", "accept", "uuid=%s" % cluster_uuid])
        self.assertThat(
            output, Contains("Nodegroup(s) accepted."))

    def test_update_dns_config(self):
        #XXX: matsubara Could be asked by maas-dns package and configurable
        # through debconf seed file.
        dns_config = open("/etc/bind/named.conf.options", 'w')
        dns_config.write(LAB_DNS_CONFIG)
        dns_config.close()
        cmd = ["service", "bind9", "restart"]
        expected_output = '* Starting domain name service... bind9'
        assertCommandReturnCode(self, cmd, expected_output)

    def power_on(self, ip, user, password):
        cmd =  ["ipmipower", "-h", ip, "-u", user, "-p", password, "--on"]
        expected_output = "%s: ok" % ip
        assertCommandReturnCode(self, cmd, expected_output)

    def power_off(self, ip, user, password):
        cmd =  ["ipmipower", "-h", ip, "-u", user, "-p", password, "--off"]
        expected_output = "%s: ok" % ip
        assertCommandReturnCode(self, cmd, expected_output)

    def _boot_nodes(self):
        # Run ipmipower to boot up nodes.
        for ipmi_address in LENOVO_LAB.values():
            self.power_off(ipmi_address, POWER_USER, POWER_PASS)
            self.power_on(ipmi_address, POWER_USER, POWER_PASS)
        if USE_ARM_NODES:
            for ipmi_address in ARM_LAB.values():
                self.power_off(ipmi_address, 'admin', 'admin')
                self.power_on(ipmi_address, 'admin', 'admin')

    # XXX: matsubara bug=1108319
    # revert the timeout back to the original value once bug is fixed.
    #@timeout(3*60)
    @timeout(8*60)
    def test_boot_nodes_enlist(self):
        if USE_CC_NODES:
            # XXX: matsubara bug=1108319
            #  The maas-cli doesn't have a way to tell if cluster controllers
            #  are missing the boot image.
            while not BootImage.objects.all().exists():
                self.addDetail(
                    "Waiting cluster controller import boot images.",
                    text_content(BootImage.objects.all()))
                sleep(10)
        self._boot_nodes()

    def _wait_nodes(self, status, min_node=None):
        """Wait for `min_node` nodes with status `status`."""
        if min_node is None:
            min_node = self.get_node_count()
        # XXX: matsubara Can't filter by status.
        output, err = self._run_maas_cli(["nodes", "list"])
        node_list = loads(output)
        filtered_list = [
            node for node in node_list if node['status'] == status]
        while not len(filtered_list) == min_node:
            sleep(5)
            output, err = self._run_maas_cli(["nodes", "list"])
            node_list = loads(output)
            filtered_list = [
                node for node in node_list if node['status'] == status]

    @timeout(7*60)
    def test_check_nodes_declared(self):
        self._wait_nodes(0)

    def test_set_nodes_ipmi_config(self):
        """Set IPMI configuration for each node."""
        all_nodes = {}
        all_nodes.update(LENOVO_LAB)
        all_nodes.update(ARM_LAB)
        for mac in all_nodes.keys():
            # run maas-cli command to search node by mac and return system_id
            out, err = self._run_maas_cli(
                ["nodes", "list", "mac_address=%s" % mac])
            node_list = loads(out)
            for node in node_list:
                if ARM_LAB.has_key(mac):
                    power_user = power_pass = 'admin'
                else:
                    power_user = POWER_USER
                    power_pass = POWER_PASS
                self._run_maas_cli(
                    ["node", "update", node['system_id'], "power_type=ipmi",
                    "power_parameters_power_address=%s " % all_nodes[mac],
                    "power_parameters_power_user=%s" % power_user,
                    "power_parameters_power_pass=%s" % power_pass])

    def test_commission_nodes(self):
        # Use maas-cli to accept all nodes.
        output, err = self._run_maas_cli(["nodes","accept-all"])
        for node in loads(output):
            self.assertEqual(node['status'], 1)

    @timeout(10*60)
    def test_check_nodes_ready(self):
        self._wait_nodes(4)

    def test_apply_tag_to_all_nodes(self):
        # Use maas-cli to set a tag on all nodes.
        output, err = self._run_maas_cli(["tags", "new", "name=all",
            "definition=true()", "comment=A tag present on all nodes"])
        tag = loads(output)
        self.assertEqual(tag['name'], "all")

    @timeout(10*60)
    def test_check_tag_applied_to_all_nodes(self):
        # Wait for all nodes to have new tag applied.
        expected_node_count = self.get_node_count()
        while True:
            output, err = self._run_maas_cli(["tag", "nodes", "all"])
            nodes = loads(output)
            if len(nodes) == expected_node_count:
                break
            sleep(5)

    def _run_juju_command(self, args):
        command = ["juju", "--debug"]
        command.extend(args)
        retcode, output, err = run_command(command)
        command_name = " ".join(map(pipes.quote, command))
        self.addDetail(command_name, text_content(str(output)))
        self.addDetail(command_name, text_content(str(err)))
        return output

    def test_configure_juju(self):
        setup_deb_proxy()
        setup_ssh()
        token_str = get_token_str()
        # Workaround bug 972829 (in juju precise).
        server_url = MAAS_URL.replace('/MAAS', ':80/MAAS')
        setup_juju_config(server_url, token_str, series=NODE_SERIES)
        setup_local_dns()

    def get_juju_status(self):
        status_output = self._run_juju_command(["status"])
        status = yaml.safe_load(status_output)
        return status

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(50*60)
    def test_juju_bootstrap(self):
        # Wait a bit to let all the nodes go down.
        # XXX: rvb 2013-04-23 bug=1171418
        sleep(30)
        if is_juju_core():
            command = ['bootstrap', '--upload-tools']
        else:
            command = ['bootstrap']
        self._run_juju_command(command)
        self._wait_machines_running(1)

    def _wait_machines_running(self, nb_machines):
        """Wait until at least `nb_machines` have their agent running."""
        while True:
            status = self.get_juju_status()
            if status is not None:
                machines = status['machines'].values()
                running_machines = [
                    machine for machine in machines
                    if machine.get('agent-state', '') in [
                        'running', 'started']]
                if len(running_machines) >= nb_machines:
                    break
            sleep(20)

    def _wait_units_started(self, service, nb_units):
        """Wait until a service has at least `nb_units` units."""
        while True:
            status = self.get_juju_status()
            try:
                units = status['services'][service]['units'].values()
            except KeyError:
                units = []
            started_units = [
                unit for unit in units
                if unit.get('agent-state', '') == 'started']
            if len(started_units) >= nb_units:
                break
            sleep(20)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    def test_juju_setup(self):
        # Deploy mediawiki with its mysql db.
        # Deploy mysql.
        self._run_juju_command(["deploy", "mysql"])
        # Deploy mediawiki.
        constraints = []
        if USE_ARM_NODES:
            constraints.append("arch=arm")
        if not is_juju_core():
            constraints.append("maas-tags=all")
        deploy_command = ["deploy", "mediawiki"]
        if len(constraints) != 0:
            deploy_command.append("--constraints")
            deploy_command.append(' '.join(constraints))
        self._run_juju_command(deploy_command)
        # Bind the services together.
        self._run_juju_command(["add-relation", "mediawiki:db", "mysql"])
        self._run_juju_command(["expose", "mediawiki"])
        # Add new unit for test 18: we run that here to
        # parallelize the installation of the new machine.
        self._run_juju_command(["add-unit", "mediawiki"])

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(40*60)
    def test_juju_deploy_mediawiki(self):
        self._wait_machines_running(3)
        self._wait_units_started('mediawiki', 1)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(6*60)
    def test_mediawiki_homepage(self):
        while True:
            status = self.get_juju_status()
            mediawiki_address = (
                status['services']['mediawiki']['units'].values(
                    )[0]['public-address'])
            self.addDetail('juju status', text_content(str(status)))
            mediawiki_url = 'http://%s/mediawiki/' % mediawiki_address
            opener = urllib2.build_opener()
            try:
                homepage = opener.open(mediawiki_url).read()
            except urllib2.URLError:
                pass
            else:
                temporary_failures = (
                    'database error' in homepage or
                    "LocalSettings.php not found" in homepage
                    )
                if not temporary_failures:
                    break
            sleep(15)

        self.assertIn("<title>Please set name of wiki</title>", homepage)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(30*60)
    def test_juju_add_unit_mediawiki(self):
        self._wait_machines_running(4)
        self._wait_units_started('mediawiki', 2)

    @classmethod
    def _collect_logs(cls, log_dirs, log_dest):
        """Collect logs from the test run.

        This method copies logs from the test run to a known location so
        auto-package-testing can copy them out of the testbed.
        """
        for log_dir in log_dirs:
            ignored, tail = os.path.split(log_dir)
            try:
                copytree(log_dir, os.path.join(log_dest, tail))
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
                # Directory does not exist: ignore and carry on.

    @classmethod
    def tearDownClass(cls):
        """Collect logs and signal to the cluster tests finished."""
        log_dest = '/var/tmp/testresults/maas-logs/'
        log_dirs = [
            '/var/log/maas',
            '/var/log/upstart',
            '/var/log/apache2',
            '/var/lib/maas/dhcp']
        cls._collect_logs(log_dirs, log_dest)
        change_logs_permissions(log_dest)

        # Signal to the cluster that the region controller tests finished.
        if not USE_CC_NODES:
            raise SkipTest("Not testing Cluster controller")
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.connect('tcp://%s:5555' % CLUSTER_CONTROLLER_IP)
        socket.send("Region controller tests finished.")