~1chb1n/mojo/eoan

# Copyright 2014 Canonical Ltd.  This software is licensed under the
# GNU General Public License version 3 (see the file LICENSE).
import subprocess
import os
import logging
import subprocess
from . import charm_repo
import re
import shutil
import tempfile
import time
import string
import sys
import yaml
from .utils import (
    bicommand,
    chdir,
    get_dep_prog,
    secs2human,
)
from mojo.exceptions import ConfigNotFoundException
from jinja2 import Template
from jinja2.exceptions import TemplateSyntaxError
import mojo
import mojo.juju
import mojo.juju.utils
import codetree
import six
from deployer.config import ConfigStack


class NoSuchPhaseError(Exception):
    pass


class FileDoesNotExistException(Exception):
    pass


class NonNumericException(Exception):
    pass


class JujuDeployerException(Exception):
    pass


class CharmAuditPhaseException(Exception):
    pass


class CollectPhaseException(Exception):
    pass


class InvalidYAMLException(Exception):
    pass


class InvalidStopOnReturnCodeException(Exception):
    pass


class StopOnPhaseException(Exception):
    pass


class VerifyPhaseException(Exception):
    pass


class NagiosCheckPhaseException(Exception):
    pass


class InvalidCollectException(Exception):
    pass


class ScriptPhaseException(Exception):
    def __init__(self, returncode, output):
        self.returncode = returncode
        self.output = output


class Phase(object):
    "Base class for phases"
    name = "not_implemented"

    def __init__(self, **kwargs):
        super(Phase, self).__init__()
        # check for config= then script= then name of phase
        self.config_base = kwargs.get('config',
                                      kwargs.get('script', self.name))
        self.options = kwargs

    def __str__(self):
        return "<{} {} {} {}>".format(self.__class__.__name__, self.name,
                                      self.config_base, self.options)

    def run(self, project, workspace, stage=None):
        raise NotImplementedError


class SleepPhase(Phase):
    """Sleep for time in seconds
       Arguments:
           config      number of seconds to sleep
                       Default: 30
    """
    name = "sleep"

    def run(self, project, workspace, stage=None):
        "Sleep for time in seconds"
        logging.debug("### Running phase {} with options {} ###"
                      "".format(self.name, self.options))
        if self.config_base == 'sleep':
            self.config_base = 30
        logging.info("Sleeping for {} seconds".format(self.config_base))
        try:
            self.config_base = int(self.config_base)
        except ValueError:
            raise NonNumericException("Time for sleep phase must be the "
                                      "numeric number of seconds not '{}'"
                                      "".format(self.config_base))
        time.sleep(self.config_base)


class CommentPhase(Phase):
    """Include a well-deliniated comment in the info logger.

    When writing a spec, comments in a manifest are useful not only when
    reading the manifest, but also in the output when running the manifest.

    For convenience, any line(s) in a manifest beginning with a single
    hash will be converted to a comment. If you require a comment in
    the manifest which isn't useful during running, use a double-hash.

    This phase cannot be run from the command line.

    Arguments:
        text      The text to be displayed.
                  Default: "comment"
    """
    name = "comment"

    def run(self, *args, **kwargs):
        border = "#" * 77
        comment = "\n".join([border, self.options['text'], border])
        logging.info("Manifest comment:\n\n" + comment + "\n\n")


class SecretsPhase(Phase):
    """Pull Secrets into workspace local
       Arguments:
           config      Directory where secrets are located
                       Default: /srv/mojo/LOCAL/$MOJO_PROJECT/$MOJO_STAGE
    """
    name = "secrets"

    def run(self, project, workspace, stage=None):
        "Pull Secrets into workspace local"

        mojo_local_dir = os.path.join(project.mojo_root, 'LOCAL',
                                      project.name, stage or 'devel')
        logging.debug("### Running phase {} with options {} ###"
                      "".format(self.name, self.options))
        logging.info("Pulling secrets from {} to {}"
                     "".format(mojo_local_dir, workspace.local_dir))
        if os.path.isdir(mojo_local_dir):
            src_files = os.listdir(mojo_local_dir)
            for file_name in src_files:
                full_file_name = os.path.join(mojo_local_dir, file_name)
                if os.path.isfile(full_file_name):
                    shutil.copy(full_file_name, workspace.local_dir)
                elif os.path.isdir(full_file_name):
                    dest_dir = os.path.join(workspace.local_dir, file_name)
                    if os.path.exists(dest_dir):
                        shutil.rmtree(dest_dir)
                    shutil.copytree(full_file_name,
                                    os.path.join(workspace.local_dir,
                                                 file_name))
                else:
                    raise FileDoesNotExistException("{} is not a file or "
                                                    "directory"
                                                    "".format(full_file_name))
        elif 'auto_secrets' in self.options:
            logging.warn("Automatic secrets phase ran but secrets "
                         "directory {} does not exist!".format(mojo_local_dir))
        else:
            raise FileDoesNotExistException("Secrets directory {} does not "
                                            "exist and secrets phase was "
                                            "specifically requested."
                                            "".format(mojo_local_dir))


class CollectPhase(Phase):
    """Collect code and resources into workspace build directory.
       Arguments:
           config      base name for config-manager file
    """
    name = "collect"

    def run(self, project, workspace, stage=None, auto_repo=True):
        "Collect build resources using config-manager"
        start_time = time.time()
        logging.debug("### Running phase {} with options {} ###"
                      "".format(self.name, self.options))
        logging.info("Building resource tree")
        env = os.environ.copy()
        env['MOJO_SERIES'] = project.series
        env['MOJO_STAGE'] = stage or 'devel'
        expand_config_base = string.Template(self.config_base).substitute(env)
        config = workspace.spec.get_config(expand_config_base, stage)
        workers = self.options.get('workers')
        with chdir(workspace.build_dir):
            try:
                tmpl_vars = {'series': project.series}
                with tempfile.NamedTemporaryFile(mode='wt') as rendered_collect:
                    with open(config) as inp:
                        try:
                            rendered_collect.write(Template(inp.read()).render(**tmpl_vars))
                        except TemplateSyntaxError as e:
                            raise InvalidCollectException("Templating Error due to "
                                                          "invalid collect syntax file {}. "
                                                          "Likely a missing end curly bracket:\n{}".format(config, e))
                    rendered_collect.flush()
                    collect_config = codetree.config.Config(rendered_collect.name)
                    if workers:
                        try:
                            collect_success = collect_config.build(workers=workers)
                        except TypeError as e:
                            errmsg = "Error during collect phase: {} - "
                            errmsg += "latest python-codetree version "
                            errmsg += "possibly needed or 'workers' option "
                            errmsg += "should be removed"
                            raise CollectPhaseException(errmsg.format(e))
                    # XXX(aluria): backward compatibility with older python-codetree versions
                    else:
                        collect_success = collect_config.build()
                if not collect_success:
                    raise CollectPhaseException("Error during collect phase")
            except (codetree.handlers.NotSameBranch,
                    codetree.handlers.NotABranch,
                    codetree.handlers.BranchDiverged) as e:
                raise CollectPhaseException("Codetree: {}".format(e))

        if auto_repo:
            # Run Repo phase automatically
            repo = CharmRepoPhase(auto_repo=True)
            repo.run(project, workspace, stage)
        completion_time = time.time() - start_time
        logging.info("Completed collect phase in {} ({:.2f}s)"
                     .format(secs2human(completion_time), completion_time))


class InstallBuildDepsPhase(Phase):
    """Install dependencies into project container
       Arguments:
           repo_keys   comma-separated list of apt repository gpg key
                       file names
           repos       comma-separated list of apt repositories
           packages    comma-separated list of packages to insall
    """
    name = "builddeps"

    def run(self, project, workspace, stage=None):
        "Install packages"
        logging.debug("### Running phase {} with options {}"
                      "".format(self.name, self.options))
        logging.info("Installing apt repos and packages")
        utils = project.utils
        repo_keys = self.options.get('repo_keys')
        if repo_keys:
            for repo_key in repo_keys.split(","):
                repo_key = os.path.join(workspace.spec_dir, stage or 'devel',
                                        repo_key)
                utils.add_repo_key(repo_key)
        repos = self.options.get('repos')
        if repos:
            for repo in repos.split(","):
                utils.add_repo(repo)
        packages = self.options.get('packages')
        if packages:
            utils.apt_update()
            utils.install_packages(*packages.split(","))


class VolumesPhase(Phase):
    """Attach persistent volumes to services.
       Arguments:
           config    base name for volumes config file
    """
    name = "volumes"

    def run(self, project, workspace, stage=None):
        "Attach volumes to services"
        logging.debug("### Running phase {} with options {}".format(self.name,
                      self.options))
        logging.info("Mounting volumes")
        mojovol = get_dep_prog("mojo-vol")
        config = workspace.spec.get_config(self.config_base, stage)
        cmd = [mojovol, '--config', config]
        output = subprocess.check_output(cmd)
        logging.info(output)


class ScriptPhase(Phase):
    """Execute a script within the build directory.
       Arguments:
           config      base name for script file
           lxc         Boolean to use lxc or not
           debug-logs  Config file with debug-logs to look for in case of failure
           debug-logs-stages-to-exclude
                       stages that will be excluded from debug-log gathering
           KEY=VALUE   set as environment variables for use by the script
    """
    name = "script"

    def run(self, project, workspace, stage=None, lxc=False,
            network=True, auto_secrets=True, gather_debug_logs=True):
        start_time = time.time()
        if auto_secrets:
            # Run secrets phase automatically first
            secrets = mojo.phase.build('secrets', **{'auto_secrets': True})
            secrets.run(project, workspace, stage)
        logging.debug("### Running phase {} with options {}".format(self.name,
                      self.options))
        logging.info("Running script {}".format(self.config_base))
        # Override use of lxc from options
        if "lxc" in self.options:
            lxc = str2bool(self.options['lxc'])

        "Run a script portion of a deployment process"
        script_env = os.environ.copy()
        script_env['MOJO_SERIES'] = project.series
        script_env['MOJO_STAGE'] = stage or 'devel'
        expand_config_base = string.Template(self.config_base).substitute(script_env)
        script = workspace.spec.get_config(expand_config_base, stage)
        # XXX: Run all scripts in a chroot or LXC environment
        env = {}
        # Set any options to the environment
        for key in self.options:
            # Parse any environment variables contained within config options
            try:
                env[key] = string.Template(self.options[key]).substitute(script_env)
            except KeyError as e:
                # We can just ignore this, as it'll be an unset env variable
                pass
        env['MOJO_PROJECT'] = project.name
        env['MOJO_WORKSPACE'] = workspace.name
        env['MOJO_SERIES'] = project.series
        env['MOJO_WORKSPACE_DIR'] = workspace.path
        env['MOJO_BUILD_DIR'] = workspace.build_dir
        env['MOJO_REPO_DIR'] = workspace.repo_dir
        env['MOJO_SPEC_DIR'] = workspace.spec_dir
        env['MOJO_LOG_DIR'] = workspace.log_dir
        env['MOJO_LOCAL_DIR'] = workspace.local_dir
        env['MOJO_STAGE'] = stage or 'devel'
        if lxc:
            project.container.run(script, env=env, network=network)
        else:
            env_vars = ["env"] + ["=".join((k, v)) for k, v in env.items()]
            command = env_vars + [script]
            try:
                output = subprocess.check_output(command, stderr=subprocess.STDOUT)
                if six.PY3:
                    output = output.decode('utf-8')
            except subprocess.CalledProcessError as e:
                if gather_debug_logs:
                    exception_output = self._gather_debug_logs(e, workspace, stage)
                else:
                    exception_output = e.output
                self._exec_on_failure(workspace, stage, env_vars)
                raise ScriptPhaseException(e.returncode, exception_output)
            if output.strip():
                logging.info(output)
        completion_time = time.time() - start_time
        logging.info("Completed script {} in {} ({:.2f}s)"
                     .format(self.config_base, secs2human(completion_time), completion_time))

    def _gather_debug_logs(self, e, workspace, stage):
        "Gather debug logs"
        dl = mojo.juju.DebugLogs(
            workspace, stage, self.options.get('debug-logs'), self.options.get('debug-logs-stages-to-exclude'))
        if dl.has_config:
            exception_output = "{}\n{}".format(e.output, dl.get_log_output())
        else:
            exception_output = e.output
        return exception_output

    def _exec_on_failure(self, workspace, stage, env_vars):
        "Confirm if we have a script to run in case of failure and run it"
        try:
            exec_on_failure_script = workspace.spec.get_config(self.options.get('exec-on-failure', 'exec-on-failure'), stage)
            logging.error("Running exec-on-failure {}".format(exec_on_failure_script))
            command = env_vars + [exec_on_failure_script]
            try:
                logging.error(subprocess.check_output(command, stderr=subprocess.STDOUT))
            except subprocess.CalledProcessError as e:
                logging.error("There was an error running exec-on-failure")
                logging.error(e.output)
        except ConfigNotFoundException:
            logging.debug("No exec-on-failure config found, see https://mojo.canonical.com/readme.html#exec-on-failure "
                          "for more details")


class BuildPhase(ScriptPhase):
    """Execute a build script within a build directory. The build environment
       is restricted to have no access to networks.
       Arguments:
           config     Base name for build script file
    """
    name = "build"

    def run(self, project, workspace, stage, auto_secrets=True):
        "Run a build script"
        start_time = time.time()
        # XXX: Remove network access during build phase
        super(BuildPhase, self).run(project, workspace, stage,
                                    lxc=True, network=False,
                                    auto_secrets=auto_secrets,
                                    gather_debug_logs=False)
        completion_time = time.time() - start_time
        logging.info("Completed build phase in {} ({:.2f}s)"
                     .format(secs2human(completion_time), completion_time))


class CharmRepoPhase(Phase):
    """Build a charm repo from paths within the build directory.
       Arguments:
           config    Base name for the charm repo spec
    """
    name = "repo"

    def run(self, project, workspace, stage):
        "Build a charm repository"
        logging.debug("### Running phase {} with options {}"
                      "".format(self.name, self.options))
        logging.info("Build a charm repository")
        repo = charm_repo.CharmRepo(workspace.repo_dir)
        if self.options.get('auto_repo'):
            logging.info("Creating charm repo automatically from all charms "
                         "in the build directory.")
            repo.build_all(workspace.build_dir, project.series)
        else:
            logging.warn("It is no longer necessary to specify a repo phase "
                         "in the manifest nor have a repo config file in the "
                         "spec. The repo phase will run automatically after "
                         "each collect phase and will copy over all charms "
                         "from the build directory to the charms directory.")
            config = workspace.spec.get_config(self.config_base, stage)
            repo.build_from_config(config, workspace.build_dir, project.series)


def identical_but_wrapped_in_newlines(current, proposed):
    """If CURRENT is the same as PROPOSED, but PROPOSED merely has
    one or more extra newlines at the start or end, return True.
    Otherwise, return False."""

    try:
        # We might have been passed a string...
        return current == proposed.strip('\n')
    except AttributeError:
        # ...and we might not.
        return current == proposed


def process_config_changes_from_diff(deployer_diff, preview=True):
    diffs = yaml.safe_load(deployer_diff)
    if not diffs:
        return

    services = diffs.get('services')
    if not services:
        return

    modified = services.get('modified')
    if not modified:
        return

    for service, changes in modified.items():
        deployer_config_items = changes.get('cfg-config')
        if not deployer_config_items:
            # Something differs, e.g. constraints, but not the config.
            continue

        juju_set_command = ['juju', mojo.juju.utils.get_juju_command('set'), service]
        config_changes = []

        for config_item, config_value in deployer_config_items.items():
            # At some point, "juju set" started trimming newlines,
            # which I endorse; but we may get a false positive here.
            if identical_but_wrapped_in_newlines(
                    changes['env-config'][config_item],
                    config_value):
                continue

            config_changes.append(
                (config_item, config_value)
            )

        if not config_changes:
            # It was all false positives, so
            # there's actually nothing to do.
            continue

        juju_set_command.extend(
            ['{}={}'.format(item, value)
             for item, value in config_changes]
        )

        if preview:
            logging.info('Preview config change (NOT RUN):\n\n{}'.format(
                ' '.join(juju_set_command),
            ))
        else:
            logging.info('Running config change for items:\n\n{}'.format(
                ' '.join([item for item, value in config_changes]),
            ))

            subprocess.check_call(juju_set_command)


class DeployerPhase(ScriptPhase):
    """Run juju-deployer.
       Arguments:
           config         Base name for deployer config. Can supply multiple times, comma separated.
           optional=y/N   When deployment fails, ignore it
           local          Base name in local dir for deployer config (secrets)
           juju           Juju environment
           delay          Deploy Delay in seconds
                          Default: 0 seconds
           status-timeout We check juju status after each deploy phase. How
                          long in seconds before we timeout if instances remain
                          in a pending status.
                          Default: 1800 seconds (30 minutes)
           target         The juju deployer configuration target to use.
           timeout        Overall timeout for the deployment
                          Default: 2700 seconds (45 minutes)
           retry          number of times to retry failed units
                          Default: unset (no retry)
           wait           Wait for the environment to reach steady state
                          Default: False
           max-wait       How long to wait for the environment to reach steady state
                          Default: 300 (5 minutes)
           additional-ready-states
                          Additional states that should be considered as ready
                          when checking juju status after a deploy phase. For
                          example: blocked,waiting
                          Default: unset (no additional states)
           status-settle-wait
                          How long to wait between successful exit of juju-deployer
                          and when the juju status check is run.  Useful for environments
                          which require additional time for resource agents to settle
                          Default: 0 (No wait)
    """
    name = "deploy"

    def _confirm_charm_repo(self, project, workspace):
        charm_repo_directory = os.path.join(workspace.repo_dir, project.series)
        if not os.path.isdir(charm_repo_directory):
            raise JujuDeployerException("Charm repository doesn't exist or isn't a directory ({}). "
                                        "You may need to run a collect phase.".format(charm_repo_directory))
        elif not os.listdir(charm_repo_directory):
            raise JujuDeployerException("Charm repository is empty ({}). You may need to run a collect phase.".format(
                charm_repo_directory))

    def _get_target(self, configs):
        """
        Return juju-deployer configuration target
        """
        deployer = get_dep_prog("juju-deployer")
        if "target" in self.options:
            target = self.options.get("target")
        else:
            # collect deployment config names and
            # and built commandline
            cmd = [deployer]
            for config in configs:
                if os.path.isfile(config):
                    cmd.extend(['-c', config])
            cmd.extend(["-l"])
            target = subprocess.check_output(cmd).split("\n")[0]
        return target

    def _validate_configs(self, configs):
        """
        Check for valid YAML or JSON configs.
        Raise exception if one t fails to parse.
        """
        for config in configs:
            try:
                with open(config) as yamlfile:
                    yaml.safe_load(yamlfile)
            except yaml.parser.ParserError as e:
                raise InvalidYAMLException("Invalid YAML or JSON in "
                                           "juju-deployer configuration file "
                                           "{}:\n{}".format(config, e))
            except yaml.scanner.ScannerError as e:
                raise InvalidYAMLException("Invalid YAML or JSON in "
                                           "juju-deployer configuration file "
                                           "{}:\n{}".format(config, e))

    def _deployer_configs(self, project, workspace, stage,
                          auto_secrets=True, cli=True):
        """
        Return list of juju deployer rendered configuration files
        If cli is True, then return list of juju deployer command line
        switches for config files:
        ('-c', '/srv/mojo/$MOJO_PROJECT/$MOJO_SPEC_DIR/$MOJO_STAGE/deploy',
         '-c', '/srv/mojo/LOCAL/$MOJO_PROJECT/$MOJO_STAGE/secrets')
        """
        if auto_secrets:
            # Run secrets phase automatically first
            secrets = SecretsPhase(auto_secrets=True)
            secrets.run(project, workspace, stage)

        # Take in MOJO_* variables from the environment
        tmpl_vars = {}
        env = os.environ.copy()
        for key in env.keys():
            if 'MOJO_' in key:
                tmpl_vars[key] = env[key]
        # Process the deployment configs as a Jinja2 template
        tmpl_vars.update({
            'project': project.name,
            'workspace': workspace.name,
            'series': project.series,
            'spec_dir': workspace.spec_dir,
            'local_dir': workspace.local_dir,
            'build_dir': workspace.build_dir,
            'stage': stage,
            'stage_name': os.path.basename(stage),
        })
        # Set arbitrary variables from options
        for key in self.options:
            if tmpl_vars.get(key):
                logging.warn("Option '{}' value '{}' overrides default '{}'"
                             "".format(key, self.options.get(key),
                                       tmpl_vars.get(key)))
            tmpl_vars[key] = self.options[key]

        configs = ()
        config_option = self.options.get('config', self.config_base).split(',')
        for config_base in config_option:
            for config_src in workspace.spec.get_configs(config_base, stage):
                # XXX Why render configs to the juju repo dir?
                config_dest = os.path.join(workspace.repo_dir, config_src)
                if not os.path.exists(os.path.dirname(config_dest)):
                    os.makedirs(os.path.dirname(config_dest))
                self.render_config(
                    os.path.join(workspace.spec_dir, config_src), config_dest,
                    tmpl_vars)
                configs += (config_dest,)

        if "local" in self.options:
            local_configs = self.options['local'].split(',')
            for config in local_configs:
                local_cfg_src = os.path.join(workspace.local_dir, config)
                if not os.path.exists(local_cfg_src):
                    raise FileDoesNotExistException(
                        "Local config file does not exist: "
                        "{}".format(local_cfg_src))
                local_cfg = os.path.join(workspace.repo_dir,
                                         "{}-local.cfg".format(
                                             os.path.basename(config)))
                self.render_config(local_cfg_src, local_cfg, tmpl_vars)
                configs += (local_cfg,)

        # Check the config files are valid YAML or JSON
        self._validate_configs(configs)

        if cli:
            cli_args = ()
            for config in configs:
                cli_args += ('-c', config)
            return cli_args
        else:
            return configs

    def show(self, project, workspace, stage):
        """
        Display rendered juju-deployer configuration files
        """
        self._confirm_charm_repo(project, workspace)
        # Quiet logging to get clean display of YAML
        logger = logging.getLogger()
        logger.setLevel(level='WARNING')
        configs = self._deployer_configs(project, workspace, stage, cli=False)
        deploy_name = self._get_target(configs)
        deployer_config = ConfigStack(configs)
        deployer_config.load()
        deployment = deployer_config.get(deploy_name)
        # XXX Pretty much a gross hack, but
        #     Config.get() forces our hand.
        deployment.repo_path = workspace.repo_dir
        deployment.resolve_config()
        print(yaml.dump(deployment.data))

    def diff(self, project, workspace, stage, diff_processor=None):
        """
        Display juju-deployer --diff comparing running environment
        to the juju-deployer configuration
        """
        self._confirm_charm_repo(project, workspace)
        deployer = get_dep_prog("juju-deployer")
        configs = self._deployer_configs(project, workspace, stage)
        deploy_name = self._get_target(configs)
        cmd = (deployer,)
        cmd += configs
        cmd += ('--diff', deploy_name)
        with chdir(workspace.repo_dir):
            if not diff_processor:
                subprocess.call(cmd)
            else:
                diff = subprocess.check_output(cmd)
                diff_processor(diff)

    def run(self, project, workspace, stage, auto_secrets=True):
        start_time = time.time()
        self._confirm_charm_repo(project, workspace)
        logging.debug("### Running phase {} with options {}"
                      "".format(self.name, self.options))
        logging.info("Running juju-deployer")
        deployer = get_dep_prog("juju-deployer")

        configs = self._deployer_configs(project, workspace, stage, auto_secrets=auto_secrets)

        juju_env = None
        if "juju" in self.options:
            juju_env = self.options['juju']
            configs += ('-e', juju_env)
        elif os.environ.get("JUJU_ENV"):
            juju_env = os.environ.get("JUJU_ENV")
            configs += ('-e', juju_env)

        if "delay" in self.options:
            deploy_delay = ('-s', str(self.options.get("delay")))
        else:
            deploy_delay = ('-s', '0')

        status_timeout = str(self.options.get("status-timeout", "1800"))

        if "timeout" in self.options:
            timeout = ('-t', str(self.options.get("timeout")))
        else:
            timeout = ('-t', '2700')

        if "retry" in self.options:
            retry = ('-r', str(self.options.get("retry")))
        else:
            retry = ()

        if "wait" in self.options:
            wait_flag = str2bool(self.options['wait'])
        else:
            wait_flag = False

        max_wait = int(self.options.get("max-wait", "300"))
        status_settle_wait = int(self.options.get("status-settle-wait", "0"))

        if "additional-ready-states" in self.options:
            additional_ready_states = self.options['additional-ready-states'].strip().split(",")
        else:
            additional_ready_states = []

        if "apply-config-changes" in self.options:
            apply_config_changes = str2bool(self.options['apply-config-changes'])
        else:
            apply_config_changes = False

        if "optional" in self.options:
            optional = str2bool(self.options['optional'])
        else:
            optional = False

        deploy_name = self._get_target(configs)

        # run the deploy
        env = os.environ.copy()
        env['MOJO_SERIES'] = project.series
        env['MOJO_STAGE'] = stage or 'devel'
        cmd = (deployer, )
        cmd += configs
        cmd += deploy_delay
        cmd += timeout
        cmd += retry
        cmd += ('-d', deploy_name, '-W', '-u')

        logging.info("Running: {}".format(" ".join(cmd)))

        if optional:
            logging.info("NOTE: This phase is marked optional; "
                         "failure will not terminate the run.")

        with chdir(workspace.repo_dir):
            status, output = bicommand(" ".join(cmd), showoutput=True, env=env)
            logging.getLogger('file').info(output)

        if status != 0:
            timeout_fail = re.search("Reached deployment timeout.. exiting",
                                     output)
            if timeout_fail:
                logging.error("Deployment timed out. It was set to {} "
                              "seconds".format(timeout[1]))
            if mojo.juju.major_version == 1:
                hook_fail = re.search(".*unit: (?P<unit>[^:]+):.*hook failed", output)
            elif mojo.juju.major_version == 2:
                hook_fail = re.search(".*unit: (?P<unit>[^:]+) .*hook failed", output)

            if hook_fail:
                failed_unit = hook_fail.group("unit")
                logging.error("Deployment failed, grabbing the last 200 lines of "
                              "output from the juju logs on {}".format(failed_unit))
                cmd = ["juju", "ssh", failed_unit, "sudo", "tail", "-200",
                       "/var/log/juju/unit-%s.log" % (
                           failed_unit.replace("/", "-"),)]
                with chdir(workspace.repo_dir):
                    failure = subprocess.check_output(cmd)
                    logging.error(failure)
            # Show juju status
            juju_status = mojo.juju.Status(environment=juju_env)
            logging.error("Juju Status: {}".format(juju_status.status()))
            if not timeout_fail and not hook_fail:
                logging.error("There was an unrecognised problem with "
                              "running a deploy phase")

            env_vars = ["env"] + ["=".join((k, v)) for k, v in env.items()]
            self._exec_on_failure(workspace, stage, env_vars)

            if optional:
                logging.info("Error found during optional deployment phase: continuing as directed.")
            else:
                raise JujuDeployerException("Error found during deployment phase")
        else:
            if status_settle_wait != 0:
                logging.info("Allowing services to settle before checking status (status_settle_wait={})".format(
                    status_settle_wait))
                time.sleep(status_settle_wait)
            if wait_flag:
                logging.info("Checking Juju status (max_wait={})".format(max_wait))
            else:
                logging.info("Checking Juju status (timeout={})".format(status_timeout))
            juju_status = mojo.juju.Status(environment=juju_env, additional_ready_states=additional_ready_states)
            try:
                juju_status.check_and_wait(timeout=status_timeout, wait_for_steady=wait_flag, max_wait=max_wait)
            except mojo.juju.JujuWaitException:
                # Show juju status
                logging.error("Juju Status: {}".format(juju_status.status()))
                raise
            if apply_config_changes:
                logging.info("Applying config changes, if any.")
                self.diff(
                    project, workspace, stage,
                    diff_processor=lambda d: process_config_changes_from_diff(
                        d, preview=False),
                )
        completion_time = time.time() - start_time
        logging.info("Completed deploy phase in {} ({:.2f}s)"
                     .format(secs2human(completion_time), completion_time))

    def render_config(self, src, dest, tmpl_vars=None):
        if not tmpl_vars:
            tmpl_vars = {}
        with open(src) as inp:
            with open(dest, "w") as outp:
                try:
                    outp.write(Template(inp.read()).render(**tmpl_vars))
                except TemplateSyntaxError as e:
                    raise InvalidYAMLException("Templating Error due to "
                                               "invalid YAML or JSON in "
                                               "juju-deployer configuration "
                                               "file {}. Likely a missing end "
                                               "curly bracket:\n{}".format(src,
                                                                           e))

        # Rendered deploy configs may contain secrets
        os.chmod(dest, 0o600)


class ApplyConfigChanges(DeployerPhase):
    name = 'apply-config-changes'

    def __init__(self, **kwargs):
        super(ApplyConfigChanges, self).__init__(**kwargs)
        # check for config= then script= then name of phase
        # XXX hack to account for subclassing; compare Phase.__init__()
        self.config_base = kwargs.get('config',
                                      kwargs.get('script', 'deploy'))

    def run(self, project, workspace, stage):
        logging.info("Applying config changes, if any.")
        DeployerPhase.diff(self, project, workspace, stage,
                           diff_processor=lambda d: process_config_changes_from_diff(d, preview=False),
                           )

    diff = run


class PreviewConfigChanges(DeployerPhase):
    name = 'preview-config-changes'

    def __init__(self, **kwargs):
        super(PreviewConfigChanges, self).__init__(**kwargs)
        # check for config= then script= then name of phase
        # XXX hack to account for subclassing; compare Phase.__init__()
        self.config_base = kwargs.get('config',
                                      kwargs.get('script', 'deploy'))

    def run(self, project, workspace, stage):
        logging.info("Previewing config changes, if any.")
        DeployerPhase.diff(self, project, workspace, stage,
                           diff_processor=process_config_changes_from_diff,
                           )

    diff = run


def int_from_dict(d, k, default):
        try:
            return int(d[k])
        except KeyError:
            pass
        except ValueError:
            logging.warning(
                "Invalid value {} for '{}', "
                "using {} instead".format(
                    d[k], k, default))

        return default


class VerifyPhase(ScriptPhase):
    """Execute a script to verify environment"
       Arguments:
           config      base name for script file
           lxc         Boolean to use lxc or not
           retry       int; how many times to retry
           sleep       int; how many seconds to wait between retries
           debug-logs  Config file with debug-logs to look for in case of failure
           debug-logs-stages-to-exclude
                       stages that will be excluded from debug-log gathering
    """
    name = "verify"

    def run(self, project, workspace, stage, auto_secrets=True):
        start_time = time.time()
        retry = int_from_dict(self.options, 'retry', 1)
        sleep = int_from_dict(self.options, 'sleep', 5)

        _run = super(VerifyPhase, self).run

        for i in range(1, retry + 1):
            if i > 1:
                logging.info(
                    "Retrying (attempt {} of {})".format(
                        i, retry))

            try:
                _run(project, workspace, stage,
                     auto_secrets=auto_secrets, gather_debug_logs=False)
            except ScriptPhaseException as e:
                if i < retry:
                    logging.warning(e.output)
                    logging.warning(
                        "Error: verify script exited with status {} "
                        "(attempt {} of {}); sleeping for {}s".format(
                            e.returncode, i, retry, sleep))
                    time.sleep(sleep)
                else:
                    dl = mojo.juju.DebugLogs(
                        workspace, stage, self.options.get('debug-logs'), self.options.get('debug-logs-stages-to-exclude'))
                    if dl.has_config:
                        exception_output = "{}\n{}".format(e.output, dl.get_log_output())
                    else:
                        exception_output = e.output
                    raise VerifyPhaseException(exception_output)
            else:
                # Hooray, success!
                break
        completion_time = time.time() - start_time
        logging.info("Completed verify phase in {} ({:.2f}s)"
                     .format(secs2human(completion_time), completion_time))


class StopOnPhase(ScriptPhase):
    """Execute a script that checks if we want to stop further execution on a
       specific return code.
       Arguments:
            config          base name for the script file
            return_code     the return code to stop further execution on
    """
    name = "stop-on"

    def run(self, project, workspace, stage):

        if "return-code" in self.options:
            return_code = int(self.options['return-code'])
            if return_code < 1 or return_code > 255:
                raise InvalidStopOnReturnCodeException("Invalid return code, must be between 1 and 255")
        else:
            raise InvalidStopOnReturnCodeException("Invalid return code, must be between 1 and 255")

        _run = super(StopOnPhase, self).run

        try:
            _run(project, workspace, stage, gather_debug_logs=False)
        except ScriptPhaseException as e:
            if e.returncode == return_code:
                # Exit silently
                logging.info(e.output)
                sys.exit(0)
            else:
                raise StopOnPhaseException(e.output)


class JujuCheckWaitPhase(Phase):
    """Check juju status and then confirm it's in a steady state

        Arguments:
            status-timeout  We check juju status after each deploy phase. How
                            long in seconds before we timeout if instances remain
                            in a pending status.
                            Default: 1800 seconds (30 minutes)
            additional-ready-states
                            Additional states that should be considered as ready
                            when checking juju status after a deploy phase. For
                            example: blocked,waiting
                            Default: unset (no additional states)
    """
    name = "juju-check-wait"

    def run(self, project, workspace, stage=None):
        status_timeout = str(self.options.get("status-timeout", "1800"))
        max_wait = self.options.get("max-wait", None)
        if max_wait is not None:
            max_wait = int(max_wait)
        additional_ready_states = self.options.get("additional-ready-states", "").strip().split(",")
        logging.info("Checking Juju status")
        juju_status = mojo.juju.Status(additional_ready_states=additional_ready_states)
        try:
            juju_status.check_and_wait(timeout=status_timeout, wait_for_steady=True, max_wait=max_wait)
        except mojo.juju.JujuWaitException:
            # Show juju status
            logging.error("Juju Status: {}".format(juju_status.status()))
            raise
        logging.info("Environment has reached steady state")


class CharmAuditPhase(Phase):
    name = "charm-audit"

    def run_charm_audit_cmd(self, machine_id):
        """Run the charm audit on a particular machine ID"""
        audit_cmd = ('for CHARMDIR in $(find /var/lib/juju/agents/unit*/ -maxdepth 1 -name charm); do '
                     'echo $(echo $CHARMDIR | cut -d\/ -f6 | sed -e "s/unit-//")":"; '
                     'echo "  application: "$(echo $CHARMDIR | cut -d\/ -f6 | sed -e "s/unit-//" '
                     '| sed -r "s/-([0-9]+)$//"); '
                     '[ -f ${CHARMDIR}/codetree-collect-info.yaml ] && '
                     'cat ${CHARMDIR}/codetree-collect-info.yaml| sed -e "s/^/  /" || true; done')
        cmd = ['juju', 'ssh', machine_id, "'{}'".format(audit_cmd), "2>/dev/null"]
        status, output = subprocess.getstatusoutput(" ".join(cmd))
        if status != 0:
            raise CharmAuditPhaseException("Unknown error trying to run {}: {}".format(" ".join(cmd), output))
        return output

    def parse_charm_audit_output(self, audit_output):
        audit_info = yaml.safe_load(audit_output)
        parsed_audit_output = {}
        for unit in list(audit_info.keys()):
            try:
                collect_url = audit_info[unit]['collect_url']
            except KeyError:
                collect_url = "Unknown"

            application = audit_info[unit]['application']

            try:
                collect_urls = list(parsed_audit_output[application].keys())
            except KeyError:
                collect_urls = []
            finally:
                if collect_url not in collect_urls:
                    parsed_audit_output[application] = {collect_url: [unit]}
                else:
                    parsed_audit_output[application][collect_url].append(unit)

        return parsed_audit_output

    def charm_audit_summary(self, parsed_audit_output):
        output = []
        for application in sorted(parsed_audit_output.keys()):
            output.append("Application: {}".format(application))
            for item in parsed_audit_output[application]:
                if item == "Unknown":
                    output.append("  Unable to determine charm version for {}".format(
                        ", ".join(parsed_audit_output[application][item])))
                else:
                    output.append("  The charm {} is deployed on {}".format(
                        item, ", ".join(parsed_audit_output[application][item])))
        return output

    def run(self, project, workspace, stage=None):
        juju_status = mojo.juju.Status()
        audit_output_list = []
        for machine_id in juju_status.machine_and_container_ids_list():
            audit_output_list.append(self.run_charm_audit_cmd(machine_id))
        parsed_audit_output = self.parse_charm_audit_output("\n".join(audit_output_list))
        print("\n".join(self.charm_audit_summary(parsed_audit_output)))


class NagiosCheckPhase(Phase):
    name = "nagios-check"

    def run_nagios_checks(self, unit, skip_checks=[]):
        """Run the nagios checks on a particular unit"""
        skip_check_command = ""
        if skip_checks:
            skip_check_command = "grep -Ev '{}' | ".format("|".join(skip_checks))
        check_cmd = "egrep -h '^command' /etc/nagios/nrpe.d/* | {} cut -d'=' -f2- | " \
                    "sed 's/.*/(set -x; & \&\& echo MOJO_NAGIOS_OK) || echo MOJO_NAGIOS_FAIL /'" \
                    "|sudo -u nagios -s bash".format(skip_check_command)
        cmd = ['juju', 'ssh', unit, '"{}"'.format(check_cmd), "2>/dev/null"]
        status, output = subprocess.getstatusoutput(" ".join(cmd))
        error_output, ok_output = "", ""
        if status != 0:
            if 'No such file or directory' in output or 'sudo: unknown user: nagios' in output:
                error_output = "Nagios not installed on {}".format(unit)
            else:
                raise NagiosCheckPhaseException("Unknown error trying to run {}: {}".format(" ".join(cmd), output))
        else:
            output_lines = [x.strip() for x in output.split("\n")]
            mojo_nagios_fail_output, mojo_nagios_all_fail_output = [], []
            mojo_nagios_ok = output_lines.count("MOJO_NAGIOS_OK")
            mojo_nagios_fail = output_lines.count("MOJO_NAGIOS_FAIL")
            if mojo_nagios_fail:
                mojo_nagios_all_fail_output.append("{} FAIL on {}:".format(mojo_nagios_fail, unit))
                for item in output_lines:
                    if item == "MOJO_NAGIOS_FAIL":
                        mojo_nagios_all_fail_output.extend(mojo_nagios_fail_output)
                    if item == "MOJO_NAGIOS_OK":
                        mojo_nagios_fail_output = []
                    else:
                        mojo_nagios_fail_output.append(item)
                error_output = "\n".join(mojo_nagios_all_fail_output)
            ok_output = "{} OK on {}".format(mojo_nagios_ok, unit)
        return error_output, ok_output

    def run(self, project, workspace, stage=None):
        retry = int_from_dict(self.options, 'retry', 1)
        sleep = int_from_dict(self.options, 'sleep', 5)

        # Which services to run the nagios checks against
        services = str(self.options.get('services', ''))
        services = [x.strip() for x in services.split(",") if x.strip()]

        # Default checks to skip:
        #  If we have any etc bzr nagios checks, we need to wait up to 15 minutes
        #  for the cron to run to populate the check file, so just ignore those.
        #  Log archive won't happen for a day after deployment.
        skip_checks = str(self.options.get("skip-checks", "check_etc_bzr,check_log_archive_status"))
        skip_checks = [x.strip() for x in skip_checks.split(",") if x.strip()]
        # User defined extra checks to skip
        skip_checks_extra = str(self.options.get("skip-checks-extra", ""))
        skip_checks.extend([x.strip() for x in skip_checks_extra.split(",") if x.strip()])
        if skip_checks:
            logging.info("Skipping the following checks: {}".format(", ".join(skip_checks)))
        else:
            logging.info("Not skipping any checks")

        for i in range(1, retry + 1):
            if i > 1:
                logging.info("Retrying (attempt {} of {})".format(i, retry))

            try:
                self._run(project, workspace, stage, services, skip_checks)
            except NagiosCheckPhaseException:
                if i < retry:
                    logging.warning(
                        "nagios-check failed (attempt {} of {}); "
                        "sleeping for {}s".format(i, retry, sleep))
                    time.sleep(sleep)
                else:
                    raise
            else:
                # Hooray, success!
                break

    def _run(self, project, workspace, stage, services, skip_checks):
        """Run the nagios-check phase"""
        juju_status = mojo.juju.Status()
        machine_ids = []
        nagios_error = False
        all_services = juju_status.services_list()
        if not services:
            services = all_services
        for service in services:
            if service not in all_services:
                logging.error("Service {} not found in {}".format(service, ", ".join(all_services)))
                nagios_error = True
                continue
            service_machine_ids = juju_status.service_machine_numbers(service)
            if service_machine_ids:
                machine_ids.extend(service_machine_ids)
                for unit in juju_status.service_units(service):
                    error_output, ok_output = self.run_nagios_checks(unit, skip_checks)
                    if ok_output:
                        logging.info(ok_output)
                    if error_output:
                        logging.error(error_output)
                        nagios_error = True
        if services == all_services:
            # We're not looking at a specific set of services, so need to warn about machines we've not checked
            for juju_status_machine_id in juju_status.machine_ids_list():
                if juju_status_machine_id not in machine_ids:
                    logging.warning("No services/nagios checks found on machine {}".format(juju_status_machine_id))
        if nagios_error:
            raise NagiosCheckPhaseException("Nagios checks failed")


# ========================================
# Phase creation helper
# ========================================

def phase_types(phase_class=None):
    "Build a dict of phases and their subtypes"
    if phase_class is None:
        phase_class = Phase
    types = {phase_class.name: phase_class}
    for subclass in phase_class.__subclasses__():
        types.update(phase_types(subclass))
    return types


def build(phase_name, **kwargs):
    "Build a phase by name, passing all arguments as a single string"
    if getattr(phase_types, "cache", None) is None:
        phase_types.cache = phase_types(Phase)
    if phase_name not in phase_types.cache:
        raise NoSuchPhaseError("{} is not a registered phase type"
                               "".format(phase_name))
    return phase_types.cache[phase_name](**kwargs)


def str2bool(v):
    if v:
        return v.lower() in ("yes", "true", "t", "1")
    return False