~andrewjbeach/juju-ci-tools/make-local-patcher

from __future__ import print_function

from collections import (

    defaultdict,

    namedtuple,

    )

from contextlib import (

    contextmanager,

    )

from copy import deepcopy

from datetime import (

    datetime,

    timedelta,

    )

import errno

from itertools import chain

import json

import logging

import os

import re

import shutil

import subprocess

import sys

from tempfile import NamedTemporaryFile

import time

import pexpect

import yaml

from jujuconfig import (

    get_environments_path,

    get_jenv_path,

    get_juju_home,

    get_selected_environment,

    )

from utility import (

    check_free_disk_space,

    ensure_deleted,

    ensure_dir,

    is_ipv6_address,

    JujuResourceTimeout,

    pause,

    quote,

    qualified_model_name,

    scoped_environ,

    skip_on_missing_file,

    split_address_port,

    temp_dir,

    unqualified_model_name,

    until_timeout,

    )

__metaclass__ = type

AGENTS_READY = set(['started', 'idle'])

WIN_JUJU_CMD = os.path.join('\\', 'Progra~2', 'Juju', 'juju.exe')

JUJU_DEV_FEATURE_FLAGS = 'JUJU_DEV_FEATURE_FLAGS'

CONTROLLER = 'controller'

KILL_CONTROLLER = 'kill-controller'

SYSTEM = 'system'

KVM_MACHINE = 'kvm'

LXC_MACHINE = 'lxc'

LXD_MACHINE = 'lxd'

_DEFAULT_BUNDLE_TIMEOUT = 3600

_jes_cmds = {KILL_CONTROLLER: {

    'create': 'create-environment',

    'kill': KILL_CONTROLLER,

}}

for super_cmd in [SYSTEM, CONTROLLER]:

    _jes_cmds[super_cmd] = {

        'create': '{} create-environment'.format(super_cmd),

        'kill': '{} kill'.format(super_cmd),

    }

log = logging.getLogger("jujupy")

class IncompatibleConfigClass(Exception):

    """Raised when a client is initialised with the wrong config class."""

class SoftDeadlineExceeded(Exception):

    """Raised when an overall client operation takes too long."""

    def __init__(self):

        super(SoftDeadlineExceeded, self).__init__(

            'Operation exceeded deadline.')

class NoProvider(Exception):

    """Raised when an environment defines no provider."""

class TypeNotAccepted(Exception):

    """Raised when the provided type was not accepted."""

class NameNotAccepted(Exception):

    """Raised when the provided name was not accepted."""

class AuthNotAccepted(Exception):

    """Raised when the provided auth was not accepted."""

def get_timeout_path():

    import timeout

    return os.path.abspath(timeout.__file__)

def get_timeout_prefix(duration, timeout_path=None):

    """Return extra arguments to run a command with a timeout."""

    if timeout_path is None:

        timeout_path = get_timeout_path()

    return (sys.executable, timeout_path, '%.2f' % duration, '--')

def get_teardown_timeout(client):

    """Return the timeout need byt the client to teardown resources."""

    if client.env.provider == 'azure':

        return 1800

    elif client.env.provider == 'gce':

        return 1200

    else:

        return 600

def parse_new_state_server_from_error(error):

    err_str = str(error)

    output = getattr(error, 'output', None)

    if output is not None:

        err_str += output

    matches = re.findall(r'Attempting to connect to (.*):22', err_str)

    if matches:

        return matches[-1]

    return None

class ErroredUnit(Exception):

    def __init__(self, unit_name, state):

        msg = '%s is in state %s' % (unit_name, state)

        Exception.__init__(self, msg)

        self.unit_name = unit_name

        self.state = state

class BootstrapMismatch(Exception):

    def __init__(self, arg_name, arg_val, env_name, env_val):

        super(BootstrapMismatch, self).__init__(

            '--{} {} does not match {}: {}'.format(

                arg_name, arg_val, env_name, env_val))

class UpgradeMongoNotSupported(Exception):

    def __init__(self):

        super(UpgradeMongoNotSupported, self).__init__(

            'This client does not support upgrade-mongo')

class JESNotSupported(Exception):

    def __init__(self):

        super(JESNotSupported, self).__init__(

            'This client does not support JES')

class JESByDefault(Exception):

    def __init__(self):

        super(JESByDefault, self).__init__(

            'This client does not need to enable JES')

class VersionNotTestedError(Exception):

    def __init__(self, version):

        super(VersionNotTestedError, self).__init__(

            'Tests for juju {} are no longer supported.'.format(version))

Machine = namedtuple('Machine', ['machine_id', 'info'])

def coalesce_agent_status(agent_item):

    """Return the machine agent-state or the unit agent-status."""

    state = agent_item.get('agent-state')

    if state is None and agent_item.get('agent-status') is not None:

        state = agent_item.get('agent-status').get('current')

    if state is None and agent_item.get('juju-status') is not None:

        state = agent_item.get('juju-status').get('current')

    if state is None:

        state = 'no-agent'

    return state

class CannotConnectEnv(subprocess.CalledProcessError):

    def __init__(self, e):

        super(CannotConnectEnv, self).__init__(e.returncode, e.cmd, e.output)

class StatusNotMet(Exception):

    _fmt = 'Expected status not reached in {env}.'

    def __init__(self, environment_name, status):

        self.env = environment_name

        self.status = status

    def __str__(self):

        return self._fmt.format(env=self.env)

class AgentsNotStarted(StatusNotMet):

    _fmt = 'Timed out waiting for agents to start in {env}.'

class VersionsNotUpdated(StatusNotMet):

    _fmt = 'Some versions did not update.'

class WorkloadsNotReady(StatusNotMet):

    _fmt = 'Workloads not ready in {env}.'

class ApplicationsNotStarted(StatusNotMet):

    _fmt = 'Timed out waiting for applications to start in {env}.'

class VotingNotEnabled(StatusNotMet):

    _fmt = 'Timed out waiting for voting to be enabled in {env}.'

@contextmanager

def temp_yaml_file(yaml_dict):

    temp_file = NamedTemporaryFile(suffix='.yaml', delete=False)

    try:

        with temp_file:

            yaml.safe_dump(yaml_dict, temp_file)

        yield temp_file.name

    finally:

        os.unlink(temp_file.name)

class SimpleEnvironment:

    """Represents a model in a JUJU_HOME directory for juju 1."""

    def __init__(self, environment, config=None, juju_home=None,

                 controller=None):

        """Constructor.

        :param environment: Name of the environment.

        :param config: Dictionary with configuration options, default is None.

        :param juju_home: Path to JUJU_HOME directory, default is None.

        :param controller: Controller instance-- this model's controller.

            If not given or None a new instance is created."""

        self.user_name = None

        if controller is None:

            controller = Controller(environment)

        self.controller = controller

        self.environment = environment

        self.config = config

        self.juju_home = juju_home

        if self.config is not None:

            try:

                provider = self.provider

            except NoProvider:

                provider = None

            self.local = bool(provider == 'local')

            self.kvm = (

                self.local and bool(self.config.get('container') == 'kvm'))

            self.maas = bool(provider == 'maas')

            self.joyent = bool(provider == 'joyent')

        else:

            self.local = False

            self.kvm = False

            self.maas = False

            self.joyent = False

    def get_option(self, key, default=None):

        return self.config.get(key, default)

    def update_config(self, new_config):

        for key, value in new_config.items():

            if key == 'region':

                logging.warning(

                    'Using set_region to set region to "{}".'.format(value))

                self.set_region(value)

                continue

            if key == 'type':

                logging.warning('Setting type is not 2.x compatible.')

            self.config[key] = value

    @property

    def provider(self):

        """Return the provider type for this environment.

        See get_cloud to determine the specific cloud.

        """

        try:

            return self.config['type']

        except KeyError:

            raise NoProvider('No provider specified.')

    def get_region(self):

        provider = self.provider

        if provider == 'azure':

            if 'tenant-id' not in self.config:

                return self.config['location'].replace(' ', '').lower()

            return self.config['location']

        elif provider == 'joyent':

            matcher = re.compile('https://(.*).api.joyentcloud.com')

            return matcher.match(self.config['sdc-url']).group(1)

        elif provider == 'lxd':

            return self.config.get('region', 'localhost')

        elif provider == 'manual':

            return self.config['bootstrap-host']

        elif provider in ('maas', 'manual'):

            return None

        else:

            return self.config['region']

    def set_region(self, region):

        try:

            provider = self.provider

        except NoProvider:

            provider = None

        if provider == 'azure':

            self.config['location'] = region

        elif provider == 'joyent':

            self.config['sdc-url'] = (

                'https://{}.api.joyentcloud.com'.format(region))

        elif provider == 'manual':

            self.config['bootstrap-host'] = region

        elif provider == 'maas':

            if region is not None:

                raise ValueError('Only None allowed for maas.')

        else:

            self.config['region'] = region

    def clone(self, model_name=None):

        config = deepcopy(self.config)

        if model_name is None:

            model_name = self.environment

        else:

            config['name'] = unqualified_model_name(model_name)

        result = self.__class__(model_name, config, self.juju_home,

                                self.controller)

        result.local = self.local

        result.kvm = self.kvm

        result.maas = self.maas

        result.joyent = self.joyent

        return result

    def __eq__(self, other):

        if type(self) != type(other):

            return False

        if self.environment != other.environment:

            return False

        if self.config != other.config:

            return False

        if self.local != other.local:

            return False

        if self.maas != other.maas:

            return False

        return True

    def __ne__(self, other):

        return not self == other

    def set_model_name(self, model_name, set_controller=True):

        if set_controller:

            self.controller.name = model_name

        self.environment = model_name

        self.config['name'] = unqualified_model_name(model_name)

    @classmethod

    def from_config(cls, name):

        """Create an environment from the configuation file.

        :param name: Name of the environment to get the configuration from."""

        return cls._from_config(name)

    @classmethod

    def _from_config(cls, name):

        config, selected = get_selected_environment(name)

        if name is None:

            name = selected

        return cls(name, config)

    def needs_sudo(self):

        return self.local

    @contextmanager

    def make_jes_home(self, juju_home, dir_name, new_config):

        """Make a JUJU_HOME/DATA directory to avoid conflicts.

        :param juju_home: Current JUJU_HOME/DATA directory, used as a

            base path for the new directory.

        :param dir_name: Name of sub-directory to make the home in.

        :param new_config: Dictionary representing the contents of

            the environments.yaml configuation file."""

        home_path = jes_home_path(juju_home, dir_name)

        with skip_on_missing_file():

            shutil.rmtree(home_path)

        os.makedirs(home_path)

        self.dump_yaml(home_path, new_config)

        # For extention: Add all files carried over to the list.

        for file_name in ['public-clouds.yaml']:

            src_path = os.path.join(juju_home, file_name)

            with skip_on_missing_file():

                shutil.copy(src_path, home_path)

        yield home_path

    def get_cloud_credentials(self):

        """Return the credentials for this model's cloud.

        This implementation returns config variables in addition to

        credentials.

        """

        return self.config

    def dump_yaml(self, path, config):

        dump_environments_yaml(path, config)

class JujuData(SimpleEnvironment):

    """Represents a model in a JUJU_DATA directory for juju 2."""

    def __init__(self, environment, config=None, juju_home=None,

                 controller=None):

        """Constructor.

        This extends SimpleEnvironment's constructor.

        :param environment: Name of the environment.

        :param config: Dictionary with configuration options; default is None.

        :param juju_home: Path to JUJU_DATA directory. If None (the default),

            the home directory is autodetected.

        :param controller: Controller instance-- this model's controller.

            If not given or None, a new instance is created.

        """

        if juju_home is None:

            juju_home = get_juju_home()

        super(JujuData, self).__init__(environment, config, juju_home,

                                       controller)

        self.credentials = {}

        self.clouds = {}

    def clone(self, model_name=None):

        result = super(JujuData, self).clone(model_name)

        result.credentials = deepcopy(self.credentials)

        result.clouds = deepcopy(self.clouds)

        return result

    @classmethod

    def from_env(cls, env):

        juju_data = cls(env.environment, env.config, env.juju_home)

        juju_data.load_yaml()

        return juju_data

    def update_config(self, new_config):

        if 'type' in new_config:

            raise ValueError('type cannot be set via update_config.')

        super(JujuData, self).update_config(new_config)

    def load_yaml(self):

        try:

            with open(os.path.join(self.juju_home, 'credentials.yaml')) as f:

                self.credentials = yaml.safe_load(f)

        except IOError as e:

            if e.errno != errno.ENOENT:

                raise RuntimeError(

                    'Failed to read credentials file: {}'.format(str(e)))

            self.credentials = {}

        self.clouds = self.read_clouds()

    def read_clouds(self):

        """Read and return clouds.yaml as a Python dict."""

        try:

            with open(os.path.join(self.juju_home, 'clouds.yaml')) as f:

                return yaml.safe_load(f)

        except IOError as e:

            if e.errno != errno.ENOENT:

                raise RuntimeError(

                    'Failed to read clouds file: {}'.format(str(e)))

            # Default to an empty clouds file.

            return {'clouds': {}}

    @classmethod

    def from_config(cls, name):

        """Create a model from the three configuration files."""

        juju_data = cls._from_config(name)

        juju_data.load_yaml()

        return juju_data

    def dump_yaml(self, path, config):

        """Dump the configuration files to the specified path.

        config is unused, but is accepted for compatibility with

        SimpleEnvironment and make_jes_home().

        """

        with open(os.path.join(path, 'credentials.yaml'), 'w') as f:

            yaml.safe_dump(self.credentials, f)

        self.write_clouds(path, self.clouds)

    @staticmethod

    def write_clouds(path, clouds):

        with open(os.path.join(path, 'clouds.yaml'), 'w') as f:

            yaml.safe_dump(clouds, f)

    def find_endpoint_cloud(self, cloud_type, endpoint):

        for cloud, cloud_config in self.clouds['clouds'].items():

            if cloud_config['type'] != cloud_type:

                continue

            if cloud_config['endpoint'] == endpoint:

                return cloud

        raise LookupError('No such endpoint: {}'.format(endpoint))

    def get_cloud(self):

        provider = self.provider

        # Separate cloud recommended by: Juju Cloud / Credentials / BootStrap /

        # Model CLI specification

        if provider == 'ec2' and self.config['region'] == 'cn-north-1':

            return 'aws-china'

        if provider not in ('maas', 'openstack'):

            return {

                'ec2': 'aws',

                'gce': 'google',

            }.get(provider, provider)

        if provider == 'maas':

            endpoint = self.config['maas-server']

        elif provider == 'openstack':

            endpoint = self.config['auth-url']

        return self.find_endpoint_cloud(provider, endpoint)

    def get_cloud_credentials_item(self):

        cloud_name = self.get_cloud()

        cloud = self.credentials['credentials'][cloud_name]

        (credentials_item,) = cloud.items()

        return credentials_item

    def get_cloud_credentials(self):

        """Return the credentials for this model's cloud."""

        return self.get_cloud_credentials_item()[1]

class StatusError(Exception):

    """Generic error for Status."""

    recoverable = True

    # This has to be filled in after the classes are declared.

    ordering = []

    @classmethod

    def priority(cls):

        """Get the priority of the StatusError as an number.

        Lower number means higher priority. This can be used as a key

        function in sorting."""

        return cls.ordering.index(cls)

class MachineError(StatusError):

    """Error in machine-status."""

    recoverable = False

class UnitError(StatusError):

    """Error in a unit's status."""

class HookFailedError(UnitError):

    """A unit hook has failed."""

    def __init__(self, item_name, msg):

        match = re.search('^hook failed: "([^"]+)"$', msg)

        if match:

            msg = match.group(1)

        super(HookFailedError, self).__init__(item_name, msg)

class InstallError(HookFailedError):

    """The unit's install hook has failed."""

    recoverable = False

class AppError(StatusError):

    """Error in an application's status."""

class AgentError(StatusError):

    """Error in a juju agent."""

class AgentUnresolvedError(AgentError):

    """Agent error has not recovered in a reasonable time."""

    # This is the time limit set by IS for recovery from an agent error.

    a_reasonable_time = timedelta(minutes=5)

    recoverable = False

StatusError.ordering = [MachineError, InstallError, AgentUnresolvedError,

                        HookFailedError, UnitError, AppError, AgentError,

                        StatusError]

class StatusItem:

    APPLICATION = 'application-status'

    WORKLOAD = 'workload-status'

    MACHINE = 'machine-status'

    JUJU = 'juju-status'

    def __init__(self, status_name, item_name, item_value):

        """Create a new StatusItem from its fields.

        :param status_name: One of the status strings.

        :param item_name: The name of the machine/unit/application the status

            information is about.

        :param item_value: A dictionary of status values. If there is an entry

            with the status_name in the dictionary its contents are used."""

        self.status_name = status_name

        self.item_name = item_name

        self.status = item_value.get(status_name, item_value)

    @property

    def message(self):

        return self.status.get('message')

    @property

    def since(self):

        return self.status.get('since')

    @property

    def current(self):

        return self.status.get('current')

    @property

    def version(self):

        return self.status.get('version')

    @property

    def datetime_since(self):

        if self.since is None:

            return None

        return datetime.strptime(self.since, '%d %b %Y %H:%M:%SZ')

    def to_exception(self):

        """Create an exception representing the error if one exists.

        :return: StatusError (or subtype) to represent an error or None

        to show that there is no error."""

        if self.current not in ['error', 'failed']:

            return None

        if self.APPLICATION == self.status_name:

            return AppError(self.item_name, self.message)

        elif self.WORKLOAD == self.status_name:

            if self.message is None:

                return UnitError(self.item_name, self.message)

            elif re.match('hook failed: ".*install.*"', self.message):

                return InstallError(self.item_name, self.message)

            elif re.match('hook failed', self.message):

                return HookFailedError(self.item_name, self.message)

            else:

                return UnitError(self.item_name, self.message)

        elif self.MACHINE == self.status_name:

            return MachineError(self.item_name, self.message)

        elif self.JUJU == self.status_name:

            time_since = datetime.utcnow() - self.datetime_since

            if time_since > AgentUnresolvedError.a_reasonable_time:

                return AgentUnresolvedError(self.item_name, self.message,

                                            time_since.total_seconds())

            else:

                return AgentError(self.item_name, self.message)

        else: