~gandelman-a/juju-deployer/darwin_redux

import logging
import subprocess
import yaml
import __builtin__
import sys
import time
import os

from os.path import isabs, join, exists
from contextlib import contextmanager
from copy import deepcopy
from base64 import b64encode


log = logging.getLogger("juju-deployer")
debug_msg = log.debug


@contextmanager
def cd(path):
    cwd = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(cwd)


def relations_combine(onto, source):
    target = deepcopy(onto)
    for (key, value) in source.items():
        if key in target:
            if isinstance(target[key], dict) and isinstance(value, dict):
                target[key] = relations_combine(target[key], value)
            elif isinstance(target[key], list) and isinstance(value, list):
                target[key] = list(set(target[key] + value))
        else:
            target[key] = value
    return target


def dict_merge(onto, source):
    target = deepcopy(onto)
    for (key, value) in source.items():
        if (key in target and isinstance(target[key], dict) and
            isinstance(value, dict)):
            if key == 'relations':
                target[key] = relations_combine(target[key], value)
            else:
                target[key] = dict_merge(target[key], value)
        else:
            target[key] = value
    return target


def init_logging(filename, debug):
    """Set up logging handlers to write messages to stdout and a file"""
    log.setLevel(logging.DEBUG)
    fh = logging.FileHandler(filename)
    fh.setLevel(logging.DEBUG)
    fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s"))
    sh = logging.StreamHandler()
    sh.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
    if not debug:
        sh.setLevel(logging.INFO)
    log.addHandler(fh)
    log.addHandler(sh)


def ensure_started(status):
    """
    ensure all service units in an environment are in the 'started' state
    """
    started = True
    for s in status["services"]:
        if 'units' not in status['services'][s]:
            continue
        for u in status["services"][s]["units"]:
            # account for recent changes in 'juju status' output
            if 'agent-state' in status["services"][s]["units"][u]:
                state = status["services"][s]["units"][u]["agent-state"]
            elif 'state' in status["services"][s]["units"][u]:
                state = status["services"][s]["units"][u]["state"]

            state = status["services"][s]["units"][u]["agent-state"]
            m_id = status["services"][s]["units"][u]["machine"]
            if "dns-name" in status["machines"][m_id].keys():
                machine = machine = status["machines"][m_id]["dns-name"]
            else:
                machine = "Pending"
            if (state == "install_error" or state == "install-error"
                or state.endswith('error')):
                log.error("Failed charm: %s, state: %s", s, state)
                exit(1)
            elif state == "started":
                pass
            else:
                started = False
            debug_msg("Service '%s'" % s)
            debug_msg(" - Machine: %s" % machine)
            debug_msg(" - State:%s" % state)
    return started


def ensure_subordinates_started(status):
    subs = []
    subs_status = []

    for s in status['services']:
        if 'subordinate' in status['services'][s]:
            subs.append(s)
            continue
        for u in status['services'][s]['units']:
            u = status['services'][s]['units'][u]
            if 'subordinates' in u:
                sub = [(u['public-address'],
                        subunit,
                        u['subordinates'][subunit]['agent-state'])
                       for subunit in u['subordinates']][0]
                subs_status.append(sub)

    started = True
    for addr, unit, state in subs_status:
        debug_msg('Subordinate unit %s' % unit)
        debug_msg(' - Machine: %s' % addr)
        debug_msg(' - State: %s' % state)

        sub = unit.split('/')[0]
        if sub in subs:
            subs.remove(sub)

        if 'error' in state:
            log.error('Failed subordinate: %s @ %s, state: %s' %
                      (unit, addr, state))
            exit(1)

        if state != 'started':
            started = False

    if subs:
        log.error('WARN: Found subordinate(s) with no principle(s): %s' % subs)

    return started


def wait_for_subordinates_started(debug, env):
    if not debug:
        sys.stdout.write('Waiting for subordinate units started')
        sys.stdout.flush()

    while ensure_subordinates_started(juju_status(env)) is False:
        if not debug:
            sys.stdout.write('.')
            sys.stdout.flush()
        time.sleep(1)
    if not debug:
        sys.stdout.write('\n')


def ensure_relations_up(status):
    """ ensure all relations are at least 'up' and error free """
    failed = []
    for s in status["services"]:
        if 'subordinate' in status['services'][s]:
            continue
        for u in status["services"][s]["units"]:
            if 'relation-errors' in status['services'][s]['units'][u]:
                errors = status['services'][s]['units'][u]['relation-errors']
                for k, v in errors.iteritems():
                    failed.append("%s:%s: %s" % (u, k, v))
    if failed:
        log.error("Failed relations: %s\n\t", "\n\t".join(failed))
        return False

    return True


def resolve_include(fname, include_dirs):
    if isabs(fname):
        return fname

    for path in include_dirs:
        full_path = join(path, fname)
        if exists(full_path):
            return full_path

    return None


def generate_deployment_config(temp, charms, include_dirs):
    """ for a given deployment, generate a deploy-time temp. config yaml for
        services that have options specified in deployments.cfg
    """
    # When resolving includes, use reverse order of the include_dirs to lookup
    # relative includes.
    include_dirs = include_dirs[::-1]

    config = {}
    for c in charms:
        cc = {}
        charm_opts = charms[c].get("options")
        charm_cfg = charms[c].get("config")
        if charm_opts is None:
            continue
        if charm_cfg is None:
            debug_msg("WARNING: Options passed to %s, but it has"
                      "no config.yaml" % (c))
            continue
        for opt, val in charm_opts.items():
            if not opt in charm_cfg.keys():
                debug_msg("WARNING: Skipping unknown config "
                          "option to %s: %s" % (c, opt))
                continue

            debug_msg("Adding option to '%s' to deploy config: %s"
                      % (opt, c))
            if (isinstance(val, basestring) and
                (val.startswith("include-file://")
                 or val.startswith("include-base64://"))):
                include, fname = val.split("://", 1)
                full_path = resolve_include(fname, include_dirs)
                if full_path is None:
                    debug_msg("WARNING: Skipping non-existing "
                              "include file %s." % fname)
                    continue
                else:
                    debug_msg("Using external file: %s for '%s'"
                              % (full_path, opt))

                with open(full_path, 'r') as f:
                    cc[opt] = f.read()

                if include.endswith("base64"):
                    cc[opt] = b64encode(cc[opt])
            else:
                cc[opt] = val

        if cc:
            config[c] = cc

    yaml.safe_dump(config, temp)
    return config


def juju_status(juju_env=None):
    """ load yaml output of 'juju status'
        retry failed 'status' calls, which sometimes seems to
        randomly happen.
    """
    def _status():
        debug_msg("Calling 'juju status'...")
        cmd = ["juju", "status"]
        if juju_env:
            cmd = cmd + ['-e', juju_env]
        p = subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        p.poll()
        (stdout, stderr) = p.communicate()
        return (stdout, stderr, p.returncode)

    max_retries = 5
    attempt = 1

    (stdout, stderr, rc) = _status()
    while rc != 0:
        if attempt == max_retries:
            log.error("Could not get Juju status after %s attempts, "
                      " giving up.", attempt)
            sys.exit(1)
        log.error("Call to 'juju status' failed!")
        log.error(stderr)
        attempt += 1
        time.sleep(3)
        (stdout, stderr, rc) = _status()
    y = yaml.load(stdout)
    return y


def juju_call(cmd, retries=5, ignore_failure=False):
    juju_log = __builtin__.juju_log

    cmd = "juju %s" % cmd
    attempt = 1

    def _call(cmd, i):
        debug_msg("Calling juju, attempt %s: %s" % (i, cmd))
        p = subprocess.Popen(cmd.split(" "),
                             stdout=juju_log,
                             stderr=juju_log)
        p.communicate()
        juju_log.flush()
        __builtin__.juju_cmds.append(cmd)
        return p.returncode

    rc = _call(cmd, attempt)
    while rc != 0:
        if ignore_failure:
            log.error("Call to '%s', ignoring." % (cmd))
            return
        if attempt == retries:
            log.error('Call to "%s" failed %s times, giving up.' %
                      (cmd, retries))
            sys.exit(1)
        attempt += 1
        log.error('Call to "%s" failed, %s attempts left.' %
                  (cmd, (retries - attempt)))
        time.sleep(3)
        rc = _call(cmd, attempt)
    return


def find_service(status, service):
    if service == "bootstrap":
        if "machines" not in status.keys():
            log.error("Environment does contain any machines?!")
            return 1
        print status["machines"][0]["dns-name"]
        return 0
    if "services" not in status.keys():
        log.error("Environment does not contain any services.")
        return 1
    if service not in status["services"].keys():
        log.error("Service '%s' not found in current deployment.", service)
        return 1
    if len(status["services"][service]["units"]) == 0:
        log.error("No units assigned to %s", service)
        return 1
    first_unit = status["services"][service]["units"].keys()[0]
    print status['services'][service]["units"][first_unit]["public-address"]
    return 0


def destroy_all(status=None, env=None, terminate_machines=False,
                scrub_zk=False, delay=None):
    """ destroy all services and optionally terminate all machines
        this is essentially 'juju destroy-environment' minus the bootstrap
        node
    """
    env_arg = ""
    if env:
        env_arg = " -e %s" % env

    destroyed = []
    if len(status["services"]) > 0:
        print "- Destroying services"
        principles = []
        subordinates = []
        for s in status['services']:
            if ('subordinate' in status['services'][s] and
                status['services'][s]['subordinate'] is True):
                subordinates.append(s)
            else:
                principles.append(s)

        def _destroy(svc):
            debug_msg("Destroying service: %s" % s)
            juju_call("destroy-service %s%s" % (svc, env_arg))
            destroyed.append(svc)

        # principles must be destroyed first.
        [_destroy(svc) for svc in principles]
        [_destroy(svc) for svc in subordinates]
    else:
        debug_msg("No services to destroy.")

    if scrub_zk and destroyed:
        debug_msg('Cleaning cached charms from ZK.')
        scrub_zookeeper(destroyed)

    if len(status["machines"]) > 0 and terminate_machines is True:
        print "- Destroying machines"
        machines = status["machines"].keys()
        for m in machines[1:]:
            debug_msg("Terminating machine: %s" % m)
            juju_call("terminate-machine %s%s" % (m, env_arg))
            if delay:
                debug_msg("Sleeping for %s between after 'terminate-machine'"
                          % delay)
                time.sleep(delay)
    elif len(status["machines"]) == 0 and terminate_machines is True:
        debug_msg("No machines to destroy")


def display_deploys(deployments):
    print "Available deployment stacks:"
    for d in sorted(deployments.keys()):
        out = "\t%s" % d
        if 'series' in deployments[d].keys():
            out += " (%s)" % deployments[d]['series']
        print out


def determine_interface(CHARMS, relation):
    """ finds the appropriate interface for a given relation
        relation = tuple (consumer, provider).  Can also return None if a
        suitable relation/interface was not found.
    """

    def _find(consumer, provider):
        if not consumer.has_key("requires"):
            return
        if not provider.has_key("provides"):
            return
        for r in consumer["requires"].keys():
            if r in provider["provides"].keys():
                if (consumer["requires"][r]["interface"] ==
                    provider["provides"][r]["interface"]):
                    debug_msg("Found interface for relation '%s': %s"
                              % (relation, r))
                    return r, r
        # Also try to match by interface type if not found by matching
        # interface name
        for cr in consumer["requires"].keys():
            for pr in  provider["provides"].keys():
                if (consumer["requires"][cr]["interface"] ==
                    provider["provides"][pr]["interface"]):
                    debug_msg("Found interface for relation '%s': %s"
                              % (relation, cr))
                    return cr, pr

    consumer = CHARMS[relation[0]]["metadata"]
    provider = CHARMS[relation[1]]["metadata"]

    # Cases we are concerned with:
    # 1) consumer/provider specified correctly
    # 2) consumer/provider specified the opposite direction
    # 3) subordinate that doesn't necessarily have an interface
    relation = _find(consumer, provider) or _find(provider, consumer)
    if relation:
        return relation
    elif is_subordinate(consumer) or is_subordinate(provider):
        return ("", "")
    else:
        log.error("WARN: No relation found? %s <=> %s -- Skipping" % (
            consumer["name"], provider["name"]))
        return


def is_subordinate(charm):
    """Given a charm, determine safely if it's a subordinate or not"""
    if not charm.has_key("subordinate"):
        return False
    return charm["subordinate"]


def ensure_interface(CHARMS, services, interface):
    """ given a specified interface, ensure they are at least
        listed in the metadata of two charms. """
    consumer = CHARMS[services[0]]["metadata"]
    provider = CHARMS[services[1]]["metadata"]
    if interface not in consumer["requires"].keys():
        log.error("Interface %s not listed as a required interface of %s",
                  services[0])
        return False
    if interface not in provider["providers"].keys():
        log.error("Interface %s not listed as a provided interface of %s",
                  services[1])
        return False
    return True


def relations_json_to_tuples(relations):
    """ convert deployment.cfg json relation configuration
        to a list of tuples that describe a relation as (consumer, provider)
    """
    t = {}
    for consumer in relations.keys():
        t[relations[consumer]["weight"]] = []
        for provider in relations[consumer]["consumes"]:
            t[relations[consumer]["weight"]].append((consumer, provider))
    return t


def wait_for_started(debug, env, msg="Waiting for all units started", sleep=1):
    if not debug:
        sys.stdout.write(msg)
        sys.stdout.flush()
    while ensure_started(juju_status(env)) is False:
        if not debug:
            sys.stdout.write(".")
            sys.stdout.flush()
        time.sleep(sleep)
    if not debug:
        sys.stdout.write("\n")


def scrub_zookeeper(deleted_services):
    """ remove charm nodes from ZK for destroyed services. useful
        when recycling a juju environment and not having to worry
        about cached charms.  this is straight lifted from
            lp:charmrunner/charmrunner/snapshot.py
        NOTE: Removing cached charms from file storage is provider-specific.
          For MAAS, remove the relevant files from /var/lib/maas/media/storage.
          Also, this needs to be run after the service is destroyed.
    """
    from twisted.internet import reactor
    from twisted.internet.defer import inlineCallbacks
    from juju.environment.config import EnvironmentsConfig
    from juju.state.service import ServiceStateManager
    import zookeeper

    env_config = EnvironmentsConfig()
    env_config.load_or_write_sample()
    if os.environ.has_key("JUJU_ENV"):
        environment = env_config.get(os.environ.get("JUJU_ENV"))
    else:
        environment = env_config.get_default()

    @inlineCallbacks
    def _clean_juju_state():
        zookeeper.set_debug_level(0)
        provider = environment.get_machine_provider()

        client = yield provider.connect()
        charms = yield client.get_children("/charms")

        # Delete any cached charm state in zookeeper
        deleted_charms = set()

        for s in deleted_services:  # XXX fuzzy match..
            for c in charms:
                if s in c:
                    deleted_charms.add(c)

        for s in (yield ServiceStateManager(client).get_all_service_states()):
            charm_id = yield s.get_charm_id()
            if charm_id in deleted_charms:
                deleted_charms.remove(charm_id)

        log.debug("Removing charms %r" % deleted_charms)
        for d in deleted_charms:
            try:
                yield client.delete("/charms/%s" % d)
            except zookeeper.NoNodeException:
                continue
        reactor.stop()

    reactor.callWhenRunning(_clean_juju_state)
    reactor.run()


def load_deployment(config, deployment):
    if deployment not in config.keys():
        log.error("deployment %s not found.", deployment)
        display_deploys(config)
        exit(1)

    deploy_config = config[deployment]
    if 'inherits' in deploy_config:
        cur = config[deployment]
        configs = []

        while 'inherits' in cur:
            configs.insert(0, cur)
            parent = cur['inherits']
            try:
                cur = config[parent]
            except KeyError:
                log.error("Could not find parent deployment in config: %s" %
                          parent)
                exit(1)

        base = cur
        configs.insert(0, base)
        configs.append(config[deployment])

        deploy_config = reduce(dict_merge, configs)

    series = deploy_config.get('series')
    charms = deploy_config.get('services', {})
    overrides = deploy_config.get('overrides', {})
    if 'relations' in deploy_config:
        relations = relations_json_to_tuples(deploy_config["relations"])
    else:
        relations = {}

    return (series, charms, relations, overrides)