~matsubara/maas/bootimages-cli

import os

from shutil import copytree

from simplejson import loads

import sys

from time import sleep

from testtools import TestCase

from testtools.matchers import Contains

from testtools.content import Content, text_content

from testtools.content_type import UTF8_TEXT

import unittest

from unittest import skipIf, SkipTest

import yaml

import platform

import urllib2

from urlparse import urlparse

import pipes

import errno

import zmq

from utils import (

    assertCommandReturnCode, assertStartedUpstartService,

    change_logs_permissions, is_juju_core, run_command, timeout,

    update_pxe_config, CLUSTER_CONTROLLER_IP, SQUID_DEB_PROXY_URL,

    HTTP_PROXY, get_maas_revision,

    )

sys.path.insert(0, "/usr/share/maas")

os.environ['DJANGO_SETTINGS_MODULE'] = 'maas.settings'

from django.core.management import call_command

from django.contrib.auth.models import User

from maasserver.models.user import get_creds_tuple

from maasserver.models import BootImage

from maasserver.utils import get_local_cluster_UUID

from apiclient.creds import convert_tuple_to_string

# Environment variables that can be used to configure

# what is tested.

# Series to install on deployed nodes.

NODE_SERIES = os.environ.get('NODE_SERIES', 'precise')

# Whether or not the ARM nodes should be used

# in the test.

DO_NOT_USE_ARM_NODES = bool(

    os.environ.get('DO_NOT_USE_ARM_NODES', False))

USE_ARM_NODES = not DO_NOT_USE_ARM_NODES

# We cannot use the ARM nodes with juju-core because of

# bug 1227722.

if is_juju_core():

    USE_ARM_NODES=False

# Whether or not the juju tests should be performed.

DO_NOT_TEST_JUJU = bool(

    os.environ.get('DO_NOT_TEST_JUJU', False))

# Whether or not set the http proxy. Currently only working on raring

DO_NOT_SET_PROXY = bool("raring" not in platform.linux_distribution())

# Whether or not the cluster controller nodes in different network

# should be used.

USE_CC_NODES = bool(os.environ.get('USE_CC_NODES', False))

MAAS_URL = "http://192.168.21.5/MAAS"

USER_DATA_URL = "http://10.98.3.59/MAAS/metadata/enlist/latest/user-data"

ADMIN_USER = "admin"

PASSWORD = "test"

POWER_USER = "root"

POWER_PASS = "ubuntu"

REGION_CONTROLLER_NODES = {

    "00:E0:81:DD:D5:99" : "192.168.22.33",

    "00:E0:81:DD:D1:0B" : "192.168.22.34",

    "00:E0:81:DD:D4:11" : "192.168.22.35",

    "00:E0:81:D1:B1:47" : "192.168.22.36",

    }

CLUSTER_CONTROLLER_NODES = {

    "00:E0:81:DD:D1:1B" : "192.168.22.37",

    "00:E0:81:DD:D1:2B" : "192.168.22.38",

    "00:E0:81:DD:D1:A3" : "192.168.22.39",

    "00:E0:81:DC:38:6D" : "192.168.22.40",

    "00:E0:81:DD:D0:FF" : "192.168.22.41",

    "00:E0:81:DD:D4:F9" : "192.168.22.42"

    }

LENOVO_LAB = REGION_CONTROLLER_NODES

if USE_CC_NODES:

    LENOVO_LAB.update(CLUSTER_CONTROLLER_NODES)

ARM_LAB = {

    'fc:2f:40:d8:fb:1a': '192.168.21.50',

    'fc:2f:40:fa:2f:9a': '192.168.21.51',

    'fc:2f:40:ec:73:42': '192.168.21.52',

    'fc:2f:40:d5:87:ce': '192.168.21.53'

    }

# dnssec is now enabled by default, so disable it to avoid errors from the

# forwarder server.

# See:

# http://stackoverflow.com/questions/13342003/ns-got-insecure-response-parent-indicates-it-should-be-secure

LAB_DNS_CONFIG = """

options {

    directory "/var/cache/bind";

    forwarders {

    10.98.3.6;

    };

    allow-query-cache {any;};

    dnssec-enable no;

    dnssec-validation no;

    auth-nxdomain no;    # conform to RFC1035

};

"""

JUJU_CONFIG = """

default: maas

environments:

  maas:

    type: maas

    maas-server: '%s'

    maas-oauth: '%s'

    admin-secret: 'nothing'

    default-series: %s

"""

def make_juju_config():

    config = JUJU_CONFIG

    if not is_juju_core():

        config = "%s%s" % (config, "    juju-origin: ppa\n")

    return config

def setup_juju_config(server, oauth, series):

    juju_dir = os.path.expanduser('~/.juju')

    try:

        os.mkdir(juju_dir)

    except OSError:

        pass

    config_content = make_juju_config() % (server, oauth, series)

    juju_configfile = os.path.join(juju_dir, 'environments.yaml')

    juju_configfile_fd = open(juju_configfile, 'w')

    juju_configfile_fd.write(config_content)

def get_token_str():

    admin = User.objects.get(username=ADMIN_USER)

    token = admin.tokens.all()[0]

    return convert_tuple_to_string(get_creds_tuple(token))

def setup_ssh():

    home_dir = os.path.expanduser('~/')

    ssh_dir = os.path.join(home_dir, '.ssh')

    # Setup ssh keys.

    ssh_key = os.path.join(ssh_dir, 'id_rsa')

    if not os.path.exists(ssh_key):

        try:

            os.mkdir(ssh_dir)

        except OSError:

            pass

        run_command(['ssh-keygen', '-t', 'rsa', '-N', '', '-f', ssh_key])

    # Setup ssh config.

    ssh_config = os.path.join(ssh_dir, 'config')

    with open(ssh_config, 'w') as f:

        f.write('StrictHostKeyChecking no')

def setup_local_dns():

    content = open('/etc/resolv.conf').read()

    content = 'nameserver 127.0.0.1\n' + content

    with open('/etc/resolv.conf', 'w') as f:

        f.write(content)

DEB_PROXY_CONFIG = """

cache_peer %s parent %s 0 no-query no-digest

never_direct allow all

""" % (urlparse(SQUID_DEB_PROXY_URL).hostname,

       urlparse(SQUID_DEB_PROXY_URL).port)

# Configure the region proxy to use the proxy.

def setup_deb_proxy():

    config = '/etc/squid-deb-proxy/squid-deb-proxy.conf'

    content = open(config).read()

    content = content + DEB_PROXY_CONFIG

    with open(config, 'w') as f:

        f.write(content)

    run_command(['sudo', 'service', 'squid-deb-proxy', 'restart'])

tests_order = [

    "test_create_admin",

    "test_restart_dbus_avahi",

    "test_update_maas_url",

    "test_restart_provisioning_server",

    "test_check_initial_services",

    "test_update_pxe_config",

    "test_import_pxe_files",

    "test_update_preseed_arm",

    "test_login_api",

    "test_set_http_proxy",

    "test_cluster_connected",

    "test_set_up_dhcp_region",

    "test_check_dhcp_service",

    "test_set_up_dhcp_cluster",

    "test_update_dns_config",

    "test_boot_nodes_enlist",

    "test_check_nodes_declared",

    "test_set_nodes_ipmi_config",

    "test_commission_nodes",

    "test_check_nodes_ready",

    "test_apply_tag_to_all_nodes",

    "test_check_tag_applied_to_all_nodes",

    "test_configure_juju",

    "test_juju_bootstrap",

    "test_juju_setup",

    "test_juju_deploy_mediawiki",

    "test_mediawiki_homepage",

    "test_juju_add_unit_mediawiki"

]

def sorting_method(ignored, first_test, second_test):

    return tests_order.index(first_test) - tests_order.index(second_test)

unittest.TestLoader.sortTestMethodsUsing = sorting_method

class TestMAASIntegration(TestCase):

    def get_node_count(self):

        """The number of available nodes."""

        count = len(LENOVO_LAB)

        if USE_ARM_NODES:

            count += len(ARM_LAB)

        return count

    def _run_maas_cli(self, args):

        retcode, output, err = run_command(["maas-cli", "maas"] + args)

        self.addDetail('retcode for maas-cli maas %s' %

            str(args), text_content(str(retcode)))

        self.addDetail('stdout for maas-cli maas %s' %

            str(args), text_content(str(output)))

        self.addDetail('stderr for maas-cli maas %s' %

            str(args), text_content(str(err)))

        return (output, err)

    def test_create_admin(self):

        """Run maas createsuperuser."""

        cmd_output = call_command(

            "createadmin", username=ADMIN_USER, password=PASSWORD,

            email="example@canonical.com", noinput=True)

        # No output, means admin user was created successfully.

        self.assertEqual(cmd_output, None)

    # Since revision 1828, MAAS doesn't use avahi/dbus anymore so

    # we do not need to restart these daemons.

    @unittest.skipIf(

        get_maas_revision() >= 1828, "Avahi/DBUS are not used anymore")

    def test_restart_dbus_avahi(self):

        cmd = ["service", "dbus", "restart"]

        expected_output = 'dbus start/running'

        assertCommandReturnCode(self, cmd, expected_output)

        cmd = ["service", "avahi-daemon", "restart"]

        expected_output = 'avahi-daemon start/running'

        assertCommandReturnCode(self, cmd, expected_output)

    def test_update_maas_url(self):

        #XXX: matsubara For some reason using a debconf file to set the

        # initial value of DEFAULT_MAAS_URL is not working.

        maas_fd = open("/etc/maas/maas_local_settings.py" , "r+")

        maas_file = maas_fd.read()

        maas_file = maas_file.replace(

            'DEFAULT_MAAS_URL = "http://10.0.2.15/MAAS"',

            'DEFAULT_MAAS_URL = "http://192.168.21.5/MAAS"')

        maas_fd.seek(0)

        maas_fd.write(maas_file)

        maas_fd.close()

        cmd = ["service", "apache2", "restart"]

        expected_output = ' * Restarting web server apache2'

        assertCommandReturnCode(self, cmd, expected_output)

    def test_restart_provisioning_server(self):

        # Restart the provisioning server as MAAS is installed before the

        # network is properly configured, which makes pserv listen only in

        # the 10.0.2.15 interface.

        cmd = ["service", "maas-pserv", "restart"]

        expected_output = 'maas-pserv start/running'

        assertCommandReturnCode(self, cmd, expected_output)

    def test_check_initial_services(self):

        assertStartedUpstartService(

            self, 'maas-cluster-celery',

            '/var/log/upstart/maas-cluster-celery.log')

        assertStartedUpstartService(

            self, 'maas-region-celery',

            '/var/log/upstart/maas-region-celery.log')

        assertStartedUpstartService(self, 'maas-txlongpoll')

        assertStartedUpstartService(

            self, 'maas-pserv', '/var/log/maas/pserv.log')

    def test_update_pxe_config(self):

        update_pxe_config()

    def test_import_pxe_files(self):

        env = os.environ.copy()

        env['http_proxy'] = HTTP_PROXY

        cmd = ['maas-import-pxe-files']

        expected_output = 'Downloading to temporary location'

        assertCommandReturnCode(self, cmd, expected_output, env=env)

    def test_update_preseed_arm(self):

        #XXX: matsubara add workaround to boot arm nodes.

        try:

            # Try the old location first, the one used before the templates

            # were moved to /etc/maas/.

            userdata_fd = open("/usr/share/maas/preseeds/enlist_userdata", "rb+")

        except IOError:

            userdata_fd = open("/etc/maas/preseeds/enlist_userdata", "rb+")

        userdata = userdata_fd.read()

        userdata += '\napt_sources:\n - source: "deb http://ports.ubuntu.com/ubuntu-ports precise-proposed main"'

        userdata_fd.seek(0)

        userdata_fd.write(userdata)

        userdata_fd.close()

    def test_login_api(self):

        token_str = get_token_str()

        api_url = MAAS_URL + "/api/1.0/"

        cmd = ["maas-cli", "login", "maas", api_url, token_str]

        expected_output = "\nYou are now logged in to the MAAS server"

        assertCommandReturnCode(self, cmd, expected_output)

    @skipIf(DO_NOT_SET_PROXY, "Not testing proxy.")

    def test_set_http_proxy(self):

        output, err = self._run_maas_cli([

            "maas", "set-config", "name=http_proxy",

            "value=%s" % SQUID_DEB_PROXY_URL])

        self.assertThat(output, Contains("OK"))

        output, err = self._run_maas_cli([

            "maas", "get-config", "name=http_proxy"])

        self.assertThat(output, Contains(SQUID_DEB_PROXY_URL))

        # Check that the user-data passed on to nodes have the proxy address.

        output = urllib2.urlopen(USER_DATA_URL).read()

        self.assertThat(output, Contains(

            "apt_proxy: %s" % SQUID_DEB_PROXY_URL))

    def get_master_ng_uuid(self):

        output, err = self._run_maas_cli(["node-groups", "list"])

        node_groups = loads(output)

        master_nodegroup = node_groups[0]

        return master_nodegroup['uuid']

    @timeout(5*60)

    def test_cluster_connected(self):

        # The master cluster is connected and changed the uuid field of the

        # nodegroup object from 'master' to its UUID.

        name = self.get_master_ng_uuid()

        while name == 'master':

            sleep(10)

            name = self.get_master_ng_uuid()

    def _update_dhcpd_apparmor_profile(self):

        """Workaround for raring due to bug 1107686."""

        dhcpd_fd = open("/etc/apparmor.d/usr.sbin.dhcpd", "r+")

        dhcpd_file = dhcpd_fd.read()

        dhcpd_file = dhcpd_file.replace(

            'network packet packet,',

            'network packet packet,\n  network packet raw,')

        dhcpd_fd.seek(0)

        dhcpd_fd.write(dhcpd_file)

        dhcpd_fd.close()

        cmd = ["service", "apparmor", "reload"]

        expected_output = '* Reloading AppArmor profiles'

        assertCommandReturnCode(self, cmd, expected_output)

    def _set_up_dhcp(self, uuid, dhcp_config):

        dhcp_cli_args = [

            "%s=%s" % (key, value) for key, value in dhcp_config.items()]

        maas_dhcp_cmd = [

            "node-group-interface", "update", uuid, "eth1" ] + dhcp_cli_args

        output, err = self._run_maas_cli(maas_dhcp_cmd)

        node_group = loads(output)

        # The JSON object returned by MAAS doesn't include the router_ip

        # address, so let's remove it from the dhcp_config before comparing.

        dhcp_config.pop('router_ip')

        self.assertEquals(node_group, dhcp_config)

    @timeout(5*60)

    def test_set_up_dhcp_region(self):

        release = platform.linux_distribution()[2]

        if release in ('quantal', 'raring'):

            self._update_dhcpd_apparmor_profile()

        region_uuid = get_local_cluster_UUID()

        output, err = self._run_maas_cli(["node-groups", "list"])

        node_groups = loads(output)

        ng_uuids = [ng['uuid'] for ng in node_groups]

        self.assertIn(region_uuid, ng_uuids)

        region_dhcp_config = {

            "ip": "192.168.21.5",

            "interface": "eth1",

            "subnet_mask": "255.255.255.0",

            "broadcast_ip": "192.168.21.255",

            "router_ip": "192.168.21.1",

            "management": 2,

            "ip_range_low": "192.168.21.10",

            "ip_range_high": "192.168.21.200"}

        self._set_up_dhcp(region_uuid, region_dhcp_config)

        # Wait for the task to complete and create the dhcpd.conf file.

        while os.path.exists("/etc/maas/dhcpd.conf") is False:

            self.addDetail(

                "Waiting task create dhcpd.conf file.",

                Content(UTF8_TEXT, lambda:

                    open("/var/log/maas/celery-region.log", 'r').readlines()))

            sleep(2)

    def test_check_dhcp_service(self):

        assertStartedUpstartService(

            self, 'maas-dhcp-server',

            '/var/log/upstart/maas-dhcp-server.log')

    @skipIf(not USE_CC_NODES, "Not testing cluster controller")

    @timeout(5*60)

    def test_set_up_dhcp_cluster(self):

        output, err = self._run_maas_cli(["node-groups", "list"])

        node_groups = loads(output)

        # Wait until we have two node_group since the cluster controller

        # might take some time to finish its configuration and contact the

        # region controller.

        while not len(node_groups) == 2:

            output, err = self._run_maas_cli(["node-groups", "list"])

            node_groups = loads(output)

        for ng in node_groups:

            if ng['uuid'] != get_local_cluster_UUID():

                cluster_uuid = ng['uuid']

                break

        # Configure the cluster controller DHCP server.

        cluster_dhcp_config = {

            "ip": "192.168.20.5",

            "interface": "eth1",

            "subnet_mask": "255.255.255.0",

            "broadcast_ip": "192.168.20.255",

            "router_ip": "192.168.20.1",

            "management": 2,

            "ip_range_low": "192.168.20.10",

            "ip_range_high": "192.168.20.30",

            }

        self._set_up_dhcp(cluster_uuid, cluster_dhcp_config)

        # Accept the cluster controller into the MAAS server.

        output, err = self._run_maas_cli([

            "node-groups", "accept", "uuid=%s" % cluster_uuid])

        self.assertThat(

            output, Contains("Nodegroup(s) accepted."))

    def test_update_dns_config(self):

        #XXX: matsubara Could be asked by maas-dns package and configurable

        # through debconf seed file.

        dns_config = open("/etc/bind/named.conf.options", 'w')

        dns_config.write(LAB_DNS_CONFIG)

        dns_config.close()

        cmd = ["service", "bind9", "restart"]

        expected_output = '* Starting domain name service... bind9'

        assertCommandReturnCode(self, cmd, expected_output)

    def power_on(self, ip, user, password):

        cmd =  ["ipmipower", "-h", ip, "-u", user, "-p", password, "--on"]

        expected_output = "%s: ok" % ip

        assertCommandReturnCode(self, cmd, expected_output)

    def power_off(self, ip, user, password):

        cmd =  ["ipmipower", "-h", ip, "-u", user, "-p", password, "--off"]

        expected_output = "%s: ok" % ip

        assertCommandReturnCode(self, cmd, expected_output)

    def _boot_nodes(self):

        # Run ipmipower to boot up nodes.

        for ipmi_address in LENOVO_LAB.values():

            self.power_off(ipmi_address, POWER_USER, POWER_PASS)

            self.power_on(ipmi_address, POWER_USER, POWER_PASS)

        if USE_ARM_NODES:

            for ipmi_address in ARM_LAB.values():

                self.power_off(ipmi_address, 'admin', 'admin')

                self.power_on(ipmi_address, 'admin', 'admin')

    # XXX: matsubara bug=1108319

    # revert the timeout back to the original value once bug is fixed.

    #@timeout(3*60)

    @timeout(8*60)

    def test_boot_nodes_enlist(self):

        if USE_CC_NODES:

            # XXX: matsubara bug=1108319

            #  The maas-cli doesn't have a way to tell if cluster controllers

            #  are missing the boot image.

            while not BootImage.objects.all().exists():

                self.addDetail(

                    "Waiting cluster controller import boot images.",

                    text_content(BootImage.objects.all()))

                sleep(10)

        self._boot_nodes()

    def _wait_nodes(self, status, min_node=None):

        """Wait for `min_node` nodes with status `status`."""

        if min_node is None:

            min_node = self.get_node_count()

        # XXX: matsubara Can't filter by status.

        output, err = self._run_maas_cli(["nodes", "list"])

        node_list = loads(output)

        filtered_list = [

            node for node in node_list if node['status'] == status]

        while not len(filtered_list) == min_node:

            sleep(5)

            output, err = self._run_maas_cli(["nodes", "list"])

            node_list = loads(output)

            filtered_list = [

                node for node in node_list if node['status'] == status]

    @timeout(7*60)

    def test_check_nodes_declared(self):

        self._wait_nodes(0)

    def test_set_nodes_ipmi_config(self):

        """Set IPMI configuration for each node."""

        all_nodes = {}

        all_nodes.update(LENOVO_LAB)

        all_nodes.update(ARM_LAB)

        for mac in all_nodes.keys():

            # run maas-cli command to search node by mac and return system_id

            out, err = self._run_maas_cli(

                ["nodes", "list", "mac_address=%s" % mac])

            node_list = loads(out)

            for node in node_list:

                if ARM_LAB.has_key(mac):

                    power_user = power_pass = 'admin'

                else:

                    power_user = POWER_USER

                    power_pass = POWER_PASS

                self._run_maas_cli(

                    ["node", "update", node['system_id'], "power_type=ipmi",

                    "power_parameters_power_address=%s " % all_nodes[mac],

                    "power_parameters_power_user=%s" % power_user,

                    "power_parameters_power_pass=%s" % power_pass])

    def test_commission_nodes(self):

        # Use maas-cli to accept all nodes.

        output, err = self._run_maas_cli(["nodes","accept-all"])

        for node in loads(output):

            self.assertEqual(node['status'], 1)

    @timeout(10*60)

    def test_check_nodes_ready(self):

        self._wait_nodes(4)

    def test_apply_tag_to_all_nodes(self):

        # Use maas-cli to set a tag on all nodes.

        output, err = self._run_maas_cli(["tags", "new", "name=all",

            "definition=true()", "comment=A tag present on all nodes"])

        tag = loads(output)

        self.assertEqual(tag['name'], "all")

    @timeout(10*60)

    def test_check_tag_applied_to_all_nodes(self):

        # Wait for all nodes to have new tag applied.

        expected_node_count = self.get_node_count()

        while True:

            output, err = self._run_maas_cli(["tag", "nodes", "all"])

            nodes = loads(output)

            if len(nodes) == expected_node_count:

                break

            sleep(5)

    def _run_juju_command(self, args):

        command = ["juju", "--debug"]

        command.extend(args)

        retcode, output, err = run_command(command)

        command_name = " ".join(map(pipes.quote, command))

        self.addDetail(command_name, text_content(str(output)))

        self.addDetail(command_name, text_content(str(err)))

        return output

    def test_configure_juju(self):

        setup_deb_proxy()

        setup_ssh()

        token_str = get_token_str()

        # Workaround bug 972829 (in juju precise).

        server_url = MAAS_URL.replace('/MAAS', ':80/MAAS')

        setup_juju_config(server_url, token_str, series=NODE_SERIES)

        setup_local_dns()

    def get_juju_status(self):

        status_output = self._run_juju_command(["status"])

        status = yaml.safe_load(status_output)

        return status

    @skipIf(DO_NOT_TEST_JUJU, "Not testing juju")

    @timeout(50*60)

    def test_juju_bootstrap(self):

        # Wait a bit to let all the nodes go down.

        # XXX: rvb 2013-04-23 bug=1171418

        sleep(30)

        if is_juju_core():

            command = ['bootstrap', '--upload-tools']

        else:

            command = ['bootstrap']

        self._run_juju_command(command)

        self._wait_machines_running(1)

    def _wait_machines_running(self, nb_machines):

        """Wait until at least `nb_machines` have their agent running."""

        while True:

            status = self.get_juju_status()

            if status is not None:

                machines = status['machines'].values()

                running_machines = [

                    machine for machine in machines

                    if machine.get('agent-state', '') in [

                        'running', 'started']]

                if len(running_machines) >= nb_machines:

                    break

            sleep(20)