~veebers/juju-ci-tools/model_migration_check_all_units_of_charm

#!/usr/bin/env python

from __future__ import print_function

from argparse import ArgumentParser

from contextlib import (

    contextmanager,

    nested,

)

from datetime import (

    datetime,

)

import errno

import glob

import logging

import os

import random

import re

import string

import subprocess

import sys

import time

import json

import shutil

from chaos import background_chaos

from fakejuju import (

    FakeBackend,

    fake_juju_client,

)

from jujucharm import (

    local_charm_path,

)

from jujuconfig import (

    get_jenv_path,

    get_juju_home,

    translate_to_env,

)

from jujupy import (

    client_from_config,

    get_local_root,

    get_machine_dns_name,

    jes_home_path,

    NoProvider,

    SimpleEnvironment,

    temp_bootstrap_env,

)

from remote import (

    remote_from_address,

    remote_from_unit,

    winrm,

)

from substrate import (

    destroy_job_instances,

    LIBVIRT_DOMAIN_RUNNING,

    resolve_remote_dns_names,

    run_instances,

    start_libvirt_domain,

    stop_libvirt_domain,

    verify_libvirt_domain,

)

from utility import (

    add_basic_testing_arguments,

    configure_logging,

    ensure_deleted,

    ensure_dir,

    logged_exception,

    LoggedException,

    PortTimeoutError,

    print_now,

    until_timeout,

    wait_for_port,

)

__metaclass__ = type

def destroy_environment(client, instance_tag):

    client.destroy_environment()

    if (client.env.provider == 'manual' and

            'AWS_ACCESS_KEY' in os.environ):

        destroy_job_instances(instance_tag)

def deploy_dummy_stack(client, charm_series):

    """"Deploy a dummy stack in the specified environment."""

    # Centos requires specific machine configuration (i.e. network device

    # order).

    if charm_series.startswith("centos") and client.env.maas:

        client.set_model_constraints({'tags': 'MAAS_NIC_1'})

    platform = 'ubuntu'

    if charm_series.startswith('win'):

        platform = 'win'

    elif charm_series.startswith('centos'):

        platform = 'centos'

    charm = local_charm_path(charm='dummy-source', juju_ver=client.version,

                             series=charm_series, platform=platform)

    client.deploy(charm, series=charm_series)

    charm = local_charm_path(charm='dummy-sink', juju_ver=client.version,

                             series=charm_series, platform=platform)

    client.deploy(charm, series=charm_series)

    client.juju('add-relation', ('dummy-source', 'dummy-sink'))

    client.juju('expose', ('dummy-sink',))

    if client.env.kvm or client.env.maas:

        # A single virtual machine may need up to 30 minutes before

        # "apt-get update" and other initialisation steps are

        # finished; two machines initializing concurrently may

        # need even 40 minutes. In addition Windows image blobs or

        # any system deployment using MAAS requires extra time.

        client.wait_for_started(7200)

    else:

        client.wait_for_started(3600)

def assess_juju_relations(client):

    token = get_random_string()

    client.set_config('dummy-source', {'token': token})

    check_token(client, token)

GET_TOKEN_SCRIPT = """

        for x in $(seq 120); do

          if [ -f /var/run/dummy-sink/token ]; then

            if [ "$(cat /var/run/dummy-sink/token)" != "" ]; then

              break

            fi

          fi

          sleep 1

        done

        cat /var/run/dummy-sink/token

        sleep 2

    """

def get_token_from_status(client):

    """Return the token from the application status message or None."""

    status = client.get_status()

    unit = status.get_unit('dummy-sink/0')

    app_status = unit.get('workload-status')

    if app_status is not None:

        message = app_status.get('message', '')

        parts = message.split()

        if parts:

            return parts[-1]

    return None

def check_token(client, token, timeout=120):

    """Check the token found on dummy-sink/0 or raise ValueError."""

    logging.info('Waiting for applications to reach ready.')

    client.wait_for_workloads()

    # Wait up to 120 seconds for token to be created.

    logging.info('Retrieving token.')

    remote = remote_from_unit(client, "dummy-sink/0")

    # Update remote with real address if needed.

    resolve_remote_dns_names(client.env, [remote])

    start = time.time()

    while True:

        if remote.is_windows():

            result = get_token_from_status(client)

            if not result:

                try:

                    result = remote.cat("%ProgramData%\\dummy-sink\\token")

                except winrm.exceptions.WinRMTransportError as e:

                    logging.warning(

                        "Skipping token check because of: {}".format(str(e)))

                    return

        else:

            result = remote.run(GET_TOKEN_SCRIPT)

        token_pattern = re.compile(r'([^\n\r]*)\r?\n?')

        result = token_pattern.match(result).group(1)

        if result == token:

            logging.info("Token matches expected %r", result)

            return

        if time.time() - start > timeout:

            if not remote.is_windows() and remote.use_juju_ssh:

                # 'juju ssh' didn't error, but try raw ssh to verify

                # the result is the same.

                remote.get_address()

                remote.use_juju_ssh = False

                result = remote.run(GET_TOKEN_SCRIPT)

                result = token_pattern.match(result).group(1)

                if result == token:

                    logging.info("Token matches expected %r", result)

                    logging.error("juju ssh to unit is broken.")

            raise ValueError('Token is %r' % result)

        logging.info("Found token %r expected %r", result, token)

        time.sleep(5)

def get_random_string():

    allowed_chars = string.ascii_uppercase + string.digits

    return ''.join(random.choice(allowed_chars) for n in range(20))

def _can_run_ssh():

    """Return true if local system can use ssh to access remote machines."""

    # When client is run on a windows machine, we have no local ssh binary.

    return sys.platform != "win32"

def dump_env_logs(client, bootstrap_host, artifacts_dir, runtime_config=None):

    if bootstrap_host is None:

        known_hosts = {}

    else:

        known_hosts = {'0': bootstrap_host}

    dump_env_logs_known_hosts(client, artifacts_dir, runtime_config,

                              known_hosts)

def dump_env_logs_known_hosts(client, artifacts_dir, runtime_config=None,

                              known_hosts=None):

    if known_hosts is None:

        known_hosts = {}

    if client.env.local:

        logging.info("Retrieving logs for local environment")

        copy_local_logs(client.env, artifacts_dir)

    else:

        remote_machines = get_remote_machines(client, known_hosts)

        for machine_id in sorted(remote_machines, key=int):

            remote = remote_machines[machine_id]

            if not _can_run_ssh() and not remote.is_windows():

                logging.info("No ssh, skipping logs for machine-%s using %r",

                             machine_id, remote)

                continue

            logging.info("Retrieving logs for machine-%s using %r", machine_id,

                         remote)

            machine_dir = os.path.join(artifacts_dir,

                                       "machine-%s" % machine_id)

            ensure_dir(machine_dir)

            copy_remote_logs(remote, machine_dir)

    archive_logs(artifacts_dir)

    retain_config(runtime_config, artifacts_dir)

def retain_config(runtime_config, log_directory):

    if not runtime_config:

        return False

    try:

        shutil.copy(runtime_config, log_directory)

        return True

    except IOError:

        print_now("Failed to copy file. Source: %s Destination: %s" %

                  (runtime_config, log_directory))

    return False

def dump_juju_timings(client, log_directory):

    try:

        with open(os.path.join(log_directory, 'juju_command_times.json'),

                  'w') as timing_file:

            json.dump(client.get_juju_timings(), timing_file, indent=2,

                      sort_keys=True)

            timing_file.write('\n')

    except Exception as e:

        print_now("Failed to save timings")

        print_now(str(e))

def get_remote_machines(client, known_hosts):

    """Return a dict of machine_id to remote machines.

    A bootstrap_host address may be provided as a fallback for machine 0 if

    status fails. For some providers such as MAAS the dns-name will be

    resolved to a real ip address using the substrate api.

    """

    # Try to get machine details from environment if possible.

    machines = dict(iter_remote_machines(client))

    # The bootstrap host is added as a fallback in case status failed.

    for machine_id, address in known_hosts.items():

        if machine_id not in machines:

            machines[machine_id] = remote_from_address(address)

    # Update remote machines in place with real addresses if substrate needs.

    resolve_remote_dns_names(client.env, machines.itervalues())

    return machines

def iter_remote_machines(client):

    try:

        status = client.get_status()

    except Exception as err:

        logging.warning("Failed to retrieve status for dumping logs: %s", err)

        return

    for machine_id, machine in status.iter_machines():

        hostname = machine.get('dns-name')

        if hostname:

            remote = remote_from_address(hostname, machine.get('series'))

            yield machine_id, remote

def archive_logs(log_dir):

    """Compress log files in given log_dir using gzip."""

    log_files = []

    for r, ds, fs in os.walk(log_dir):

        log_files.extend(os.path.join(r, f) for f in fs if is_log(f))

    if log_files:

        subprocess.check_call(['gzip', '--best', '-f'] + log_files)

def is_log(file_name):

    """Check to see if the given file name is the name of a log file."""

    return file_name.endswith('.log') or file_name.endswith('syslog')

lxc_template_glob = '/var/lib/juju/containers/juju-*-lxc-template/*.log'

def copy_local_logs(env, directory):

    """Copy logs for all machines in local environment."""

    local = get_local_root(get_juju_home(), env)

    log_names = [os.path.join(local, 'cloud-init-output.log')]

    log_names.extend(glob.glob(os.path.join(local, 'log', '*.log')))

    log_names.extend(glob.glob(lxc_template_glob))

    try:

        subprocess.check_call(['sudo', 'chmod', 'go+r'] + log_names)

        subprocess.check_call(['cp'] + log_names + [directory])

    except subprocess.CalledProcessError as e:

        logging.warning("Could not retrieve local logs: %s", e)

def copy_remote_logs(remote, directory):

    """Copy as many logs from the remote host as possible to the directory."""

    # This list of names must be in the order of creation to ensure they

    # are retrieved.

    if remote.is_windows():

        log_paths = [

            "%ProgramFiles(x86)%\\Cloudbase Solutions\\Cloudbase-Init\\log\\*",

            "C:\\Juju\\log\\juju\\*.log",

        ]

    else:

        log_paths = [

            '/var/log/cloud-init*.log',

            '/var/log/juju/*.log',

            # TODO(gz): Also capture kvm container logs?

            '/var/lib/juju/containers/juju-*-lxc-*/',

            '/var/log/lxd/juju-*',

            '/var/log/lxd/lxd.log',

            '/var/log/syslog',

            '/var/log/mongodb/mongodb.log',

            '/etc/network/interfaces',

            '/etc/environment',

            '/home/ubuntu/ifconfig.log',

        ]

        try:

            wait_for_port(remote.address, 22, timeout=60)

        except PortTimeoutError:

            logging.warning("Could not dump logs because port 22 was closed.")

            return

        try:

            remote.run('sudo chmod -Rf go+r ' + ' '.join(log_paths))

        except subprocess.CalledProcessError as e:

            # The juju log dir is not created until after cloud-init succeeds.

            logging.warning("Could not allow access to the juju logs:")

            logging.warning(e.output)

        try:

            remote.run('ifconfig > /home/ubuntu/ifconfig.log')

        except subprocess.CalledProcessError as e:

            logging.warning("Could not capture ifconfig state:")

            logging.warning(e.output)

    try:

        remote.copy(directory, log_paths)

    except (subprocess.CalledProcessError,

            winrm.exceptions.WinRMTransportError) as e:

        # The juju logs will not exist if cloud-init failed.

        logging.warning("Could not retrieve some or all logs:")

        if getattr(e, 'output', None):

            logging.warning(e.output)

        else:

            logging.warning(repr(e))

def assess_juju_run(client):

    responses = client.run(('uname',),

                           applications=['dummy-source', 'dummy-sink'])

    for machine in responses:

        if machine.get('ReturnCode', 0) != 0:

            raise ValueError('juju run on machine %s returned %d: %s' % (

                             machine.get('MachineId'),

                             machine.get('ReturnCode'),

                             machine.get('Stderr')))

    logging.info(

        "juju run succeeded on machines: %r",

        [str(machine.get("MachineId")) for machine in responses])

    return responses

def assess_upgrade(old_client, juju_path):

    all_clients = _get_clients_to_upgrade(old_client, juju_path)

    # all clients have the same provider type, work this out once.

    if all_clients[0].env.provider == 'maas':

        timeout = 1200

    else:

        timeout = 600

    for client in all_clients:

        upgrade_juju(client)

        client.wait_for_version(client.get_matching_agent_version(), timeout)

def _get_clients_to_upgrade(old_client, juju_path):

    """Return a list of cloned clients to upgrade.

    Ensure that the controller (if available) is the first client in the list.

    """

    new_client = old_client.clone_path_cls(juju_path)

    all_clients = sorted(

        new_client.iter_model_clients(),

        key=lambda m: m.model_name == 'controller',

        reverse=True)

    return all_clients

def upgrade_juju(client):

    client.set_testing_agent_metadata_url()

    tools_metadata_url = client.get_agent_metadata_url()

    logging.info(

        'The {url_type} is {url}'.format(

            url_type=client.agent_metadata_url,

            url=tools_metadata_url))

    client.upgrade_juju()

def deploy_job_parse_args(argv=None):

    parser = ArgumentParser('deploy_job')

    add_basic_testing_arguments(parser)

    parser.add_argument('--upgrade', action="store_true", default=False,

                        help='Perform an upgrade test.')

    parser.add_argument('--with-chaos', default=0, type=int,

                        help='Deploy and run Chaos Monkey in the background.')

    parser.add_argument('--jes', action='store_true',

                        help='Use JES to control environments.')

    parser.add_argument(

        '--controller-host', help=(

            'Host with a controller to use.  If supplied, SSO_EMAIL and'

            ' SSO_PASSWORD environment variables will be used for oauth'

            ' authentication.'))

    return parser.parse_args(argv)

def deploy_job():

    args = deploy_job_parse_args()

    configure_logging(args.verbose)

    series = args.series

    if series is None:

        series = 'precise'

    charm_series = series

    # Don't need windows or centos state servers.

    if series.startswith("win") or series.startswith("centos"):

        logging.info('Setting default series to trusty for win and centos.')

        series = 'trusty'

    return _deploy_job(args, charm_series, series)

def update_env(env, new_env_name, series=None, bootstrap_host=None,

               agent_url=None, agent_stream=None, region=None):

    # Rename to the new name.

    env.set_model_name(new_env_name)

    new_config = {}

    if series is not None:

        new_config['default-series'] = series

    if bootstrap_host is not None:

        new_config['bootstrap-host'] = bootstrap_host

    if agent_url is not None:

        new_config['tools-metadata-url'] = agent_url

    if agent_stream is not None:

        new_config['agent-stream'] = agent_stream

    env.update_config(new_config)

    if region is not None:

        env.set_region(region)

@contextmanager

def temp_juju_home(client, new_home):

    """Temporarily override the client's home directory."""

    old_home = client.env.juju_home

    client.env.juju_home = new_home

    try:

        yield

    finally:

        client.env.juju_home = old_home

def make_controller_strategy(client, tear_down_client, controller_host):

    if controller_host is None:

        return CreateController(client, tear_down_client)

    else:

        return PublicController(

            controller_host, os.environ['SSO_EMAIL'],

            os.environ['SSO_PASSWORD'], client, tear_down_client)

class CreateController:

    """A Controller strategy where the controller is created.

    Intended for use with BootstrapManager.

    """

    def __init__(self, client, tear_down_client):

        self.client = client

        self.tear_down_client = tear_down_client

    def prepare(self):

        """Prepare client for use by killing the existing controller."""

        self.tear_down_client.kill_controller()

    def create_initial_model(self, upload_tools, series, boot_kwargs):

        """Create the initial model by bootstrapping."""

        self.client.bootstrap(

            upload_tools=upload_tools, bootstrap_series=series,

            **boot_kwargs)

    def get_hosts(self):

        """Provide the controller host."""

        host = get_machine_dns_name(

            self.client.get_controller_client(), '0')

        if host is None:

            raise ValueError('Could not get machine 0 host')

        return {'0': host}

    def tear_down(self):

        """Tear down via client.tear_down."""

        self.tear_down_client.tear_down()

class PublicController:

    """A controller strategy where the controller is public.

    The user registers with the controller, and adds the initial model.

    """

    def __init__(self, controller_host, email, password, client,

                 tear_down_client):

        self.controller_host = controller_host

        self.email = email

        self.password = password

        self.client = client

        self.tear_down_client = tear_down_client

    def prepare(self):

        """Prepare by destroying the model and unregistering if possible."""

        try:

            self.tear_down()

        except subprocess.CalledProcessError:

            # Assume that any error tearing down means that there was nothing

            # to tear down.

            pass

    def create_initial_model(self, upload_tools, series, boot_kwargs):

        """Register controller and add model."""

        self.client.register_host(

            self.controller_host, self.email, self.password)

        self.client.env.controller.explicit_region = True

        self.client.add_model(self.client.env)

    def get_hosts(self):

        """There are no user-owned controller hosts, so no-op."""

        return {}

    def tear_down(self):

        """Remove the current model and clean up the controller."""

        try:

            self.tear_down_client.destroy_model()

        finally:

            controller = self.tear_down_client.env.controller.name

            self.tear_down_client.juju('unregister', ('-y', controller),

                                       include_e=False)

class BootstrapManager:

    """

    Helper class for running juju tests.

    Enables running tests on the manual provider and on MAAS systems, with

    automatic cleanup, logging, etc.  See BootstrapManager.booted_context.

    :ivar temp_env_name: a unique name for the juju env, such as a Jenkins

        job name.

    :ivar client: an EnvJujuClient.

    :ivar tear_down_client: an EnvJujuClient for tearing down the environment

        (may be more reliable/capable/compatible than client.)

    :ivar bootstrap_host: None, or the address of a manual or MAAS host to

        bootstrap on.

    :ivar machine: [] or a list of machines to use add to a manual env

        before deploying services.

    :ivar series: None or the default-series for the temp config.

    :ivar agent_url: None or the agent-metadata-url for the temp config.

    :ivar agent_stream: None or the agent-stream for the temp config.

    :ivar log_dir: The path to the directory to store logs.

    :ivar keep_env: False or True to not destroy the environment and keep

        it alive to do an autopsy.

    :ivar upload_tools: False or True to upload the local agent instead of

        using streams.

    :ivar known_hosts: A dict mapping machine_ids to hosts for

        dump_env_logs_known_hosts.

    """

    def __init__(self, temp_env_name, client, tear_down_client, bootstrap_host,

                 machines, series, agent_url, agent_stream, region, log_dir,

                 keep_env, permanent, jes_enabled, controller_strategy=None):

        """Constructor.

        Please see see `BootstrapManager` for argument descriptions.

        """

        self.temp_env_name = temp_env_name

        self.bootstrap_host = bootstrap_host

        self.machines = machines

        self.series = series

        self.agent_url = agent_url

        self.agent_stream = agent_stream

        self.region = region

        self.log_dir = log_dir

        self.keep_env = keep_env

        if jes_enabled and not permanent:

            raise ValueError('Cannot set permanent False if jes_enabled is'

                             ' True.')

        self.permanent = permanent

        self.jes_enabled = jes_enabled

        self.known_hosts = {}

        if bootstrap_host is not None:

            self.known_hosts['0'] = bootstrap_host

        if controller_strategy is None:

            controller_strategy = CreateController(client, tear_down_client)

        self.controller_strategy = controller_strategy

        self.has_controller = False

    @property

    def client(self):

        return self.controller_strategy.client

    @property

    def tear_down_client(self):

        return self.controller_strategy.tear_down_client

    @classmethod

    def _generate_default_clean_dir(cls, temp_env_name):

        """Creates a new unique directory for logging and returns name"""

        logging.info('Environment {}'.format(temp_env_name))