~matsubara/maas/dhcpd-workaround-raring

import os
from simplejson import loads
import sys
from subprocess import Popen, PIPE
from time import sleep
from testtools import TestCase
from testtools.matchers import Contains, Equals, StartsWith
from testtools.content import text_content
from unittest import skipIf
import yaml
import platform
import urllib2

from timeout import timeout

sys.path.insert(0, "/usr/share/maas")
os.environ['DJANGO_SETTINGS_MODULE'] = 'maas.settings'
from django.core.management import call_command
from django.contrib.auth.models import User

from maasserver.models.user import get_creds_tuple
from apiclient.creds import convert_tuple_to_string

# Environment variables that can be used to configured
# what is tested.

# Series to install on deployed nodes.
NODE_SERIES = os.environ.get('NODE_SERIES', 'precise')

# Whether or not the ARM nodes should be used
# in the test.
DO_NOT_USE_ARM_NODES = bool(
    os.environ.get('DO_NOT_USE_ARM_NODES', False))


USE_ARM_NODES = not DO_NOT_USE_ARM_NODES


# Whether or not the juju tests should be performed.
DO_NOT_TEST_JUJU = bool(
    os.environ.get('DO_NOT_TEST_JUJU', False))


MAAS_URL = "http://192.168.21.5/MAAS"
ADMIN_USER = "admin"
PASSWORD = "test"
POWER_USER = "root"
POWER_PASS = "ubuntu"

LENOVO_LAB = {
    "00:E0:81:DD:D5:99" : "192.168.22.33",
    "00:E0:81:DD:D1:0B" : "192.168.22.34",
    "00:E0:81:DD:D4:11" : "192.168.22.35",
    "00:E0:81:D1:B1:47" : "192.168.22.36",
# There are in a different network, enable them
# when testing with a CC on a different network
# is supported.
#    "00:E0:81:DD:D1:1B" : "192.168.22.37",
#    "00:E0:81:DD:D1:2B" : "192.168.22.38",
#    "00:E0:81:DD:D1:A3" : "192.168.22.39",
#    "00:E0:81:DC:38:6D" : "192.168.22.40",
#    "00:E0:81:DD:D0:FF" : "192.168.22.41",
#    "00:E0:81:DD:D4:F9" : "192.168.22.42"
    }

ARM_LAB = {
    'fc:2f:40:d8:fb:1a': '192.168.21.50',
    'fc:2f:40:fa:2f:9a': '192.168.21.51',
    'fc:2f:40:ec:73:42': '192.168.21.52',
    'fc:2f:40:d5:87:ce': '192.168.21.53'
    }

LAB_DNS_CONFIG = """
options {
    directory "/var/cache/bind";
    forwarders {
    10.98.0.1;
    };
    allow-query-cache {any;};
    dnssec-validation auto;
    auth-nxdomain no;    # conform to RFC1035
};
"""

JUJU_CONFIG = """
default: maas
environments:
  maas:
    type: maas
    maas-server: '%s'
    maas-oauth: '%s'
    admin-secret: 'nothing'
    default-series: %s
    juju-origin: ppa
"""


def setup_juju_config(server, oauth, series):
    juju_dir = os.path.expanduser('~/.juju')
    try:
        os.mkdir(juju_dir)
    except OSError:
        pass
    config_content = JUJU_CONFIG % (server, oauth, series)
    juju_configfile = os.path.join(juju_dir, 'environments.yaml')
    juju_configfile_fd = open(juju_configfile, 'w')
    juju_configfile_fd.write(config_content)


def get_token_str():
    admin = User.objects.get(username=ADMIN_USER)
    token = admin.tokens.all()[0]
    return convert_tuple_to_string(get_creds_tuple(token))


def setup_ssh():
    home_dir = os.path.expanduser('~/')
    ssh_dir = os.path.join(home_dir, '.ssh')
    # Setup ssh keys.
    ssh_key = os.path.join(ssh_dir, 'id_rsa')
    if not os.path.exists(ssh_key):
        try:
            os.mkdir(ssh_dir)
        except OSError:
            pass
        Popen(
            ['ssh-keygen', '-t', 'rsa', '-N', '', '-f', ssh_key],
            stdout=PIPE, stderr=PIPE).communicate()
    # Setup ssh config.
    ssh_config = os.path.join(ssh_dir, 'config')
    with open(ssh_config, 'w') as f:
        f.write('StrictHostKeyChecking no')


def setup_local_dns():
    content = open('/etc/resolv.conf').read()
    content = 'nameserver 127.0.0.1\n' + content
    with open('/etc/resolv.conf', 'w') as f:
        f.write(content)


DEB_PROXY_CONFIG = """
cache_peer 10.98.0.13 parent 8000 0 no-query no-digest
never_direct allow all
"""


# Configure the region proxy to use the proxy on 10.98.0.13.
def setup_deb_proxy():
    config = '/etc/squid-deb-proxy/squid-deb-proxy.conf'
    content = open(config).read()
    content = content + DEB_PROXY_CONFIG
    with open(config, 'w') as f:
        f.write(content)
    Popen(
        ['sudo', 'service', 'squid-deb-proxy', 'restart'],
        stdout=PIPE, stderr=PIPE).communicate()


class TestMAASIntegration(TestCase):

    def get_node_count(self):
        """The number of available nodes."""
        count = len(LENOVO_LAB)
        if USE_ARM_NODES:
            count += len(ARM_LAB)
        return count

    def _run_command(self, args):
        process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE)
        stdout, stderr = process.communicate()
        return stdout, stderr

    def _run_maas_cli(self, args):
        result = self._run_command(["maas-cli", "maas"] + args)
        self.addDetail(
            'maas-cli maas %s' % str(args), text_content(str(result)))
        return result

    def test_00_disable_raring(self):
        """Disable raring, the images are not available yet.
        # XXX: rvb 2013-02-04 bug=1115178
        # XXX: rvb 2013-02-04 bug=1115175
        """
        maas_fd = open("/etc/maas/import_pxe_files" , "r+")
        maas_file = maas_fd.read()
        maas_file = maas_file.replace(
            'RELEASES="precise quantal raring"',
            'RELEASES="precise quantal"')
        maas_fd.seek(0)
        maas_fd.write(maas_file)
        maas_fd.close()

    def test_01_create_admin(self):
        """Run maas createsuperuser."""
        cmd_output = call_command(
            "createadmin", username=ADMIN_USER, password=PASSWORD,
            email="example@canonical.com", noinput=True)
        # No output, means admin user was created successfully.
        self.assertEqual(cmd_output, None)

    def test_02_update_maas_url(self):
        #XXX: matsubara For some reason using a debconf file to set the
        # initial value of DEFAULT_MAAS_URL is not working.
        maas_fd = open("/etc/maas/maas_local_settings.py" , "r+")
        maas_file = maas_fd.read()
        maas_file = maas_file.replace(
            'DEFAULT_MAAS_URL = "http://10.98.0.90/MAAS"',
            'DEFAULT_MAAS_URL = "http://192.168.21.5/MAAS"')
        maas_fd.seek(0)
        maas_fd.write(maas_file)
        maas_fd.close()
        output, err = self._run_command(["service", "apache2", "restart"])
        self.assertThat(
            output, StartsWith(' * Restarting web server apache2'))

    def test_03_restart_dbus_avahi(self):
        """XXX: matsubara bug=1065775 """
        output, err = self._run_command(["service", "dbus", "restart"])
        self.assertThat(output, Contains('dbus start/running'))
        output, err = self._run_command(["service", "avahi-daemon", "restart"])
        self.assertThat(output, Contains('avahi-daemon start/running'))

    def test_04_update_pxe_config(self):
        ephemerals_fd = open("/etc/maas/import_ephemerals", "r+")
        ephemerals = ephemerals_fd.read()
        # Update mirrors for both trunk and the quantal SRU.
        ephemerals_snippet = (
            '\n'
            'CLOUD_IMAGES_ARCHIVE="http://10.98.0.13/mirrors/maas-ephemeral"\n'
            'REMOTE_IMAGES_MIRROR="http://10.98.0.13/mirrors/maas-ephemeral"\n'
        #    XXX: rvb 2013-01-18: Do not use the daily ephemeral images as
        #    they seem to be broken (more precisely, the image from 20130107 is
        #    and the one from 20121008 isn't);  investigation is underway.
        #   'STREAM=daily\n'
            )
        ephemerals += ephemerals_snippet
        ephemerals_fd.seek(0)
        ephemerals_fd.write(ephemerals)
        ephemerals_fd.close()
        # XXX: matsubara Bug=1074167
        # Update import_pxe_files to not download squashfs images
        # and use a proxy.
        pxe_fd = open('/etc/maas/import_pxe_files', "r+")
        pxe_file = pxe_fd.read()
        pxe_snippet = 'export http_proxy="http://10.98.0.13:3128"\n'
        pxe_file = pxe_snippet + pxe_file
        pxe_fd.seek(0)
        pxe_fd.write(pxe_file)
        pxe_fd.close()

    def test_05_import_pxe_files(self):
        output, err = self._run_command(["maas-import-pxe-files"])
        self.assertThat(output, Contains('Downloading to temporary location'))
        # XXX: matsubara Bug=1076444
        # maas-import-pxe-files outputs to stderr during normal operation.
        #self.assertIs(err, '')

    def test_06_update_preseed_arm(self):
        #XXX: matsubara add workaround to boot arm nodes.
        userdata_fd = open("/usr/share/maas/preseeds/enlist_userdata", "r+")
        userdata = userdata_fd.read()
        userdata += '\napt_sources:\n - source: "deb http://ports.ubuntu.com/ubuntu-ports precise-proposed main"'
        userdata_fd.seek(0)
        userdata_fd.write(userdata)
        userdata_fd.close()

    def test_07_login_api(self):
        token_str = get_token_str()
        api_url = MAAS_URL + "/api/1.0/"
        output, err = self._run_command(
            ["maas-cli", "login", "maas", api_url, token_str])
        self.assertThat(
            output, Contains(
            "\nYou are now logged in to the MAAS server"))

    def get_master_ng_uuid(self):
        output, err = self._run_maas_cli(["node-groups", "list"])
        node_groups = loads(output)
        master_nodegroup = node_groups[0]
        return master_nodegroup['uuid']

    @timeout(5*60)
    def test_08_cluster_connected(self):
        # The master cluster is connected and changed the uuid field of the
        # nodegroup object from 'master' to its UUID.
        name = self.get_master_ng_uuid()
        while name == 'master':
            sleep(10)
            name = self.get_master_ng_uuid()

    def _update_dhcpd_apparmor_profile(self):
        """Workaround for raring due to bug 1107686."""
        dhcpd_fd = open("/etc/apparmor.d/usr.sbin.dhcpd", "r+")
        dhcpd_file = dhcpd_fd.read()
        dhcpd_file = dhcpd_file.replace(
            'network packet packet,',
            'network packet packet,\n  network packet raw,')
        dhcpd_fd.seek(0)
        dhcpd_fd.write(dhcpd_file)
        dhcpd_fd.close()
        output, err = self._run_command(["service", "apparmor", "reload"])
        self.assertThat(output, Contains('* Reloading AppArmor profiles'))

    @timeout(60)
    def test_09_set_up_dhcp(self):
        if "raring" in platform.linux_distribution():
            self._update_dhcpd_apparmor_profile()

        output, err = self._run_maas_cli(["node-groups", "list"])
        node_groups = loads(output)
        output, err = self._run_maas_cli([
            "node-group-interface", "update", node_groups[0]['uuid'],
            "eth1", "ip=192.168.21.5", "interface=eth1", "management=2",
            "subnet_mask=255.255.255.0",
            "broadcast_ip=192.168.21.255",
            "router_ip=192.168.21.1",
            "ip_range_low=192.168.21.10",
            "ip_range_high=192.168.21.30"])
        node_group = loads(output)
        self.assertThat(node_group['ip'], Equals('192.168.21.5'))
        self.assertThat(node_group['interface'], Equals('eth1'))
        self.assertThat(node_group['subnet_mask'], Equals('255.255.255.0'))
        self.assertThat(node_group['broadcast_ip'], Equals('192.168.21.255'))
        self.assertThat(node_group['ip_range_low'], Equals('192.168.21.10'))
        self.assertThat(node_group['ip_range_high'], Equals('192.168.21.30'))
        # Wait for the task to complete and create the dhcpd.conf file.
        while os.path.exists("/etc/maas/dhcpd.conf") is False:
            sleep(2)

    def test_10_update_dns_config(self):
        #XXX: matsubara Could be asked by maas-dns package and configurable
        # through debconf seed file.
        dns_config = open("/etc/bind/named.conf.options", 'w')
        dns_config.write(LAB_DNS_CONFIG)
        dns_config.close()
        output, err = self._run_command(["service", "bind9", "restart"])
        self.assertThat(output, Contains(
            '* Starting domain name service... bind9'))

    def power_on(self, ip, user, password):
        output, err = self._run_command(
            ["ipmipower", "-h", ip, "-u", user, "-p", password, "--on"])
        self.addDetail('IPMI power on %s' % ip, text_content(str(output)))
        self.assertThat(output, Contains("%s: ok" % ip))

    def power_off(self, ip, user, password):
        output, err = self._run_command(
            ["ipmipower", "-h", ip, "-u", user, "-p", password, "--off"])
        self.addDetail('IPMI power off %s' % ip, text_content(str(output)))
        self.assertThat(output, Contains("%s: ok" % ip))

    def _boot_nodes(self):
        # Run ipmipower to boot up nodes.
        for ipmi_address in LENOVO_LAB.values():
            self.power_off(ipmi_address, POWER_USER, POWER_PASS)
            self.power_on(ipmi_address, POWER_USER, POWER_PASS)
        if USE_ARM_NODES:
            for ipmi_address in ARM_LAB.values():
                self.power_off(ipmi_address, 'admin', 'admin')
                self.power_on(ipmi_address, 'admin', 'admin')

    def test_11_boot_nodes_enlist(self):
        self._boot_nodes()

    def _wait_nodes(self, status, min_node=None):
        """Wait for `min_node` nodes with status `status`."""
        if min_node is None:
            min_node = self.get_node_count()
        # XXX: matsubara Can't filter by status.
        output, err = self._run_maas_cli(["nodes", "list"])
        node_list = loads(output)
        filtered_list = [
            node for node in node_list if node['status'] == status]
        while not len(filtered_list) == min_node:
            sleep(5)
            output, err = self._run_maas_cli(["nodes", "list"])
            node_list = loads(output)
            filtered_list = [
                node for node in node_list if node['status'] == status]

    @timeout(5*60)
    def test_12_check_nodes_declared(self):
        self._wait_nodes(0)

    def test_13_set_nodes_ipmi_config(self):
        """Set IPMI configuration for each node."""
        all_nodes = {}
        all_nodes.update(LENOVO_LAB)
        all_nodes.update(ARM_LAB)
        for mac in all_nodes.keys():
            # run maas-cli command to search node by mac and return system_id
            out, err = self._run_maas_cli(
                ["nodes", "list", "mac_address=%s" % mac])
            node_list = loads(out)
            for node in node_list:
                if ARM_LAB.has_key(mac):
                    power_user = power_pass = 'admin'
                else:
                    power_user = POWER_USER
                    power_pass = POWER_PASS
                self._run_maas_cli(
                    ["node", "update", node['system_id'], "power_type=ipmi",
                    "power_parameters_power_address=%s " % all_nodes[mac],
                    "power_parameters_power_user=%s" % power_user,
                    "power_parameters_power_pass=%s" % power_pass])

    def test_14_commission_nodes(self):
        # Use maas-cli to accept all nodes.
        output, err = self._run_maas_cli(["nodes","accept-all"])
        for node in loads(output):
            self.assertEqual(node['status'], 1)

    @timeout(10*60)
    def test_15_check_nodes_ready(self):
        self._wait_nodes(4)

    def setup_juju(self):
        setup_deb_proxy()
        setup_ssh()
        token_str = get_token_str()
        # Workaround bug 972829 (in juju precise).
        server_url = MAAS_URL.replace('/MAAS', ':80/MAAS')
        setup_juju_config(server_url, token_str, series=NODE_SERIES)
        setup_local_dns()

    def _run_juju_command(self, args):
        output, err = self._run_command(["juju"] + args)
        # For some reason, in the log, juju replaces '-' by '_'.
        command_name = args[0].replace('-', '_')
        self.assertIn(
            "INFO '%s' command finished successfully" % command_name, err)
        return output

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(50*60)
    def test_16_juju_bootstrap(self):
        self.setup_juju()
        self._run_juju_command(['bootstrap'])
        self._run_juju_command(['status'])

    def get_juju_status(self):
        status_output = self._run_juju_command(["status"])
        self.addDetail('juju status', text_content(str(status_output)))
        status = yaml.load(status_output)
        return status

    def _wait_machines_running(self, nb_machines):
        """Wait until at least `nb_machines` have their agent running."""
        while True:
            status = self.get_juju_status()
            machines = status['machines'].values()
            running_machines = [
                machine for machine in machines
                if machine.get('agent-state', '') == 'running']
            if len(running_machines) >= nb_machines:
                break
            sleep(20)

    def _wait_units_started(self, service, nb_units):
        """Wait until a service has at least `nb_units` units."""
        while True:
            status = self.get_juju_status()
            try:
                units = status['services'][service]['units'].values()
            except KeyError:
                units = []
            started_units = [
                unit for unit in units
                if unit.get('agent-state', '') == 'started']
            if len(started_units) >= nb_units:
                break
            sleep(20)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    def test_17_juju_setup(self):
        # Deploy mediawiki with its mysql db.
        self._run_juju_command(["deploy", "mysql"])
        is_precise = platform.linux_distribution()[2] == 'precise'
        if USE_ARM_NODES and not is_precise:
            self._run_juju_command(
                ["deploy", "mediawiki", "--constraints", "arch=arm"])
        else:
            self._run_juju_command(["deploy", "mediawiki"])
        self._run_juju_command(["add-relation", "mediawiki:db", "mysql"])
        self._run_juju_command(["expose", "mediawiki"])
        # Add new unit for test 18: we run that here to
        # parallelize the installation of the new machine.
        self._run_juju_command(["add-unit", "mediawiki"])

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(30*60)
    def test_18_juju_deploy_mediawiki(self):
       self._wait_machines_running(3)
       self._wait_units_started('mediawiki', 1)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(5*60)
    def test_19_mediawiki_homepage(self):
        while True:
            status = self.get_juju_status()
            mediawiki_address = (
                status['services']['mediawiki']['units'].values(
                    )[0]['public-address'])
            self.addDetail('juju status', text_content(str(status)))
            mediawiki_url = 'http://%s/mediawiki/' % mediawiki_address
            opener = urllib2.build_opener()
            try:
                homepage = opener.open(mediawiki_url).read()
            except urllib2.URLError:
                pass
            else:
                temporary_failures = (
                    'database error' in homepage or
                    "LocalSettings.php not found" in homepage
                    )
                if not temporary_failures:
                    break
            sleep(15)

        self.assertIn("<title>Please set name of wiki</title>", homepage)

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")
    @timeout(30*60)
    def test_20_juju_add_unit_mediawiki(self):
        self._wait_machines_running(4)
        self._wait_units_started('mediawiki', 2)