~hduran-8/juju-ci-tools/add_storage_ci_tests

__metaclass__ = type

from collections import defaultdict

from contextlib import contextmanager

from datetime import (

    datetime,

    timedelta,

)

import os

import shutil

import StringIO

import subprocess

import tempfile

from textwrap import dedent

from unittest import TestCase

from mock import (

    call,

    MagicMock,

    patch,

)

import yaml

from jujuconfig import (

    get_environments_path,

    get_jenv_path,

    NoSuchEnvironment,

)

from jujupy import (

    CannotConnectEnv,

    Environment,

    EnvJujuClient,

    EnvJujuClient22,

    EnvJujuClient24,

    EnvJujuClient25,

    ErroredUnit,

    GroupReporter,

    get_local_root,

    JujuClientDevel,

    JUJU_DEV_FEATURE_FLAGS,

    SimpleEnvironment,

    Status,

    temp_bootstrap_env,

    _temp_env as temp_env,

    uniquify_local,

    make_client,

    parse_new_state_server_from_error,

)

from utility import (

    scoped_environ,

    temp_dir,

)

def assert_juju_call(test_case, mock_method, client, expected_args,

                     call_index=None, assign_stderr=False):

    if call_index is None:

        test_case.assertEqual(len(mock_method.mock_calls), 1)

        call_index = 0

    empty, args, kwargs = mock_method.mock_calls[call_index]

    test_case.assertEqual(args, (expected_args,))

    kwarg_keys = ['env']

    if assign_stderr:

        with tempfile.TemporaryFile() as example:

            # 'example' is a pragmatic way of checking file types in py2 and 3.

            kwarg_keys = ['stderr'] + kwarg_keys

            test_case.assertIsInstance(kwargs['stderr'], type(example))

    test_case.assertItemsEqual(kwargs.keys(), kwarg_keys)

    bin_dir = os.path.dirname(client.full_path)

    test_case.assertRegexpMatches(kwargs['env']['PATH'],

                                  r'^{}\:'.format(bin_dir))

class TestErroredUnit(TestCase):

    def test_output(self):

        e = ErroredUnit('bar', 'baz')

        self.assertEqual('bar is in state baz', str(e))

class ClientTest(TestCase):

    def setUp(self):

        super(ClientTest, self).setUp()

        patcher = patch('jujupy.pause')

        self.addCleanup(patcher.stop)

        self.pause_mock = patcher.start()

class TestEnvJujuClient25(ClientTest):

    client_class = EnvJujuClient25

    def test__shell_environ_not_cloudsigma(self):

        client = self.client_class(

            SimpleEnvironment('baz', {'type': 'ec2'}), '1.25-foobar', 'path')

        env = client._shell_environ()

        self.assertEqual(env.get(JUJU_DEV_FEATURE_FLAGS, ''), '')

    def test__shell_environ_cloudsigma(self):

        client = self.client_class(

            SimpleEnvironment('baz', {'type': 'cloudsigma'}),

            '1.25-foobar', 'path')

        env = client._shell_environ()

        "".split()

        self.assertTrue('cloudsigma' in env[JUJU_DEV_FEATURE_FLAGS].split(","))

    def test__shell_environ_juju_home(self):

        client = self.client_class(

            SimpleEnvironment('baz', {'type': 'ec2'}), '1.25-foobar', 'path')

        env = client._shell_environ(juju_home='asdf')

        self.assertEqual(env['JUJU_HOME'], 'asdf')

class TestEnvJujuClient22(ClientTest):

    client_class = EnvJujuClient22

    def test__shell_environ(self):

        client = self.client_class(

            SimpleEnvironment('baz', {'type': 'ec2'}), '1.22-foobar', 'path')

        env = client._shell_environ()

        self.assertEqual(env.get(JUJU_DEV_FEATURE_FLAGS), 'actions')

    def test__shell_environ_juju_home(self):

        client = self.client_class(

            SimpleEnvironment('baz', {'type': 'ec2'}), '1.22-foobar', 'path')

        env = client._shell_environ(juju_home='asdf')

        self.assertEqual(env['JUJU_HOME'], 'asdf')

class TestEnvJujuClient24(TestEnvJujuClient25):

    client_class = EnvJujuClient25

class TestEnvJujuClient(ClientTest):

    def test_get_version(self):

        value = ' 5.6 \n'

        with patch('subprocess.check_output', return_value=value) as vsn:

            version = EnvJujuClient.get_version()

        self.assertEqual('5.6', version)

        vsn.assert_called_with(('juju', '--version'))

    def test_get_version_path(self):

        with patch('subprocess.check_output', return_value=' 4.3') as vsn:

            EnvJujuClient.get_version('foo/bar/baz')

        vsn.assert_called_once_with(('foo/bar/baz', '--version'))

    def test_get_matching_agent_version(self):

        client = EnvJujuClient(SimpleEnvironment(None, {'type': 'local'}),

                               '1.23-series-arch', None)

        self.assertEqual('1.23.1', client.get_matching_agent_version())

        self.assertEqual('1.23', client.get_matching_agent_version(

                         no_build=True))

        client.version = '1.20-beta1-series-arch'

        self.assertEqual('1.20-beta1.1', client.get_matching_agent_version())

    def test_upgrade_juju_nonlocal(self):

        client = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'nonlocal'}), '1.234-76', None)

        with patch.object(client, 'juju') as juju_mock:

            client.upgrade_juju()

        juju_mock.assert_called_with(

            'upgrade-juju', ('--version', '1.234'))

    def test_upgrade_juju_local(self):

        client = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'local'}), '1.234-76', None)

        with patch.object(client, 'juju') as juju_mock:

            client.upgrade_juju()

        juju_mock.assert_called_with(

            'upgrade-juju', ('--version', '1.234', '--upload-tools',))

    def test_upgrade_juju_no_force_version(self):

        client = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'local'}), '1.234-76', None)

        with patch.object(client, 'juju') as juju_mock:

            client.upgrade_juju(force_version=False)

        juju_mock.assert_called_with(

            'upgrade-juju', ('--upload-tools',))

    def test_by_version(self):

        def juju_cmd_iterator():

            yield '1.17'

            yield '1.16'

            yield '1.16.1'

            yield '1.15'

            yield '1.22.1'

            yield '1.24-alpha1'

            yield '1.24.7'

            yield '1.25.1'

            yield '1.26.1'

        context = patch.object(

            EnvJujuClient, 'get_version',

            side_effect=juju_cmd_iterator().send)

        with context:

            self.assertIs(EnvJujuClient,

                          type(EnvJujuClient.by_version(None)))

            with self.assertRaisesRegexp(Exception, 'Unsupported juju: 1.16'):

                EnvJujuClient.by_version(None)

            with self.assertRaisesRegexp(Exception,

                                         'Unsupported juju: 1.16.1'):

                EnvJujuClient.by_version(None)

            client = EnvJujuClient.by_version(None)

            self.assertIs(EnvJujuClient, type(client))

            self.assertEqual('1.15', client.version)

            client = EnvJujuClient.by_version(None)

            self.assertIs(type(client), EnvJujuClient22)

            client = EnvJujuClient.by_version(None)

            self.assertIs(type(client), EnvJujuClient24)

            self.assertEqual(client.version, '1.24-alpha1')

            client = EnvJujuClient.by_version(None)

            self.assertIs(type(client), EnvJujuClient24)

            self.assertEqual(client.version, '1.24.7')

            client = EnvJujuClient.by_version(None)

            self.assertIs(type(client), EnvJujuClient25)

            self.assertEqual(client.version, '1.25.1')

            client = EnvJujuClient.by_version(None)

            self.assertIs(type(client), EnvJujuClient)

            self.assertEqual(client.version, '1.26.1')

    def test_by_version_path(self):

        with patch('subprocess.check_output', return_value=' 4.3') as vsn:

            client = EnvJujuClient.by_version(None, 'foo/bar/qux')

        vsn.assert_called_once_with(('foo/bar/qux', '--version'))

        self.assertNotEqual(client.full_path, 'foo/bar/qux')

        self.assertEqual(client.full_path, os.path.abspath('foo/bar/qux'))

    def test_full_args(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, 'my/juju/bin')

        full = client._full_args('bar', False, ('baz', 'qux'))

        self.assertEqual(('juju', '--show-log', 'bar', '-e', 'foo', 'baz',

                          'qux'), full)

        full = client._full_args('bar', True, ('baz', 'qux'))

        self.assertEqual((

            'juju', '--show-log', 'bar', '-e', 'foo',

            'baz', 'qux'), full)

        client.env = None

        full = client._full_args('bar', False, ('baz', 'qux'))

        self.assertEqual(('juju', '--show-log', 'bar', 'baz', 'qux'), full)

    def test_full_args_debug(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, 'my/juju/bin')

        client.debug = True

        full = client._full_args('bar', False, ('baz', 'qux'))

        self.assertEqual((

            'juju', '--debug', 'bar', '-e', 'foo', 'baz', 'qux'), full)

    def test_full_args_action(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, 'my/juju/bin')

        full = client._full_args('action bar', False, ('baz', 'qux'))

        self.assertEqual((

            'juju', '--show-log', 'action', 'bar', '-e', 'foo', 'baz', 'qux'),

            full)

    def test_bootstrap_hpcloud(self):

        env = Environment('hp', '')

        with patch.object(env, 'hpcloud', lambda: True):

            with patch.object(EnvJujuClient, 'juju') as mock:

                EnvJujuClient(env, None, None).bootstrap()

            mock.assert_called_with(

                'bootstrap', ('--constraints', 'mem=2G'), False,

                juju_home=None)

    def test_bootstrap_maas(self):

        env = Environment('maas', '')

        with patch.object(EnvJujuClient, 'juju') as mock:

            client = EnvJujuClient(env, None, None)

            with patch.object(client.env, 'maas', lambda: True):

                client.bootstrap()

            mock.assert_called_with(

                'bootstrap', ('--constraints', 'mem=2G arch=amd64'), False,

                juju_home=None)

    def test_bootstrap_joyent(self):

        env = Environment('joyent', '')

        with patch.object(EnvJujuClient, 'juju', autospec=True) as mock:

            client = EnvJujuClient(env, None, None)

            with patch.object(client.env, 'joyent', lambda: True):

                client.bootstrap()

            mock.assert_called_once_with(

                client, 'bootstrap', ('--constraints', 'mem=2G cpu-cores=1'),

                False, juju_home=None)

    def test_bootstrap_non_sudo(self):

        env = Environment('foo', '')

        with patch.object(EnvJujuClient, 'juju') as mock:

            client = EnvJujuClient(env, None, None)

            with patch.object(client.env, 'needs_sudo', lambda: False):

                client.bootstrap()

            mock.assert_called_with(

                'bootstrap', ('--constraints', 'mem=2G'), False,

                juju_home=None)

    def test_bootstrap_sudo(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client.env, 'needs_sudo', lambda: True):

            with patch.object(client, 'juju') as mock:

                client.bootstrap()

            mock.assert_called_with(

                'bootstrap', ('--constraints', 'mem=2G'), True, juju_home=None)

    def test_bootstrap_upload_tools(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client.env, 'needs_sudo', lambda: True):

            with patch.object(client, 'juju') as mock:

                client.bootstrap(upload_tools=True)

            mock.assert_called_with(

                'bootstrap', ('--upload-tools', '--constraints', 'mem=2G'),

                True, juju_home=None)

    def test_bootstrap_juju_home(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client.env, 'needs_sudo', lambda: True):

            with patch.object(client, 'juju', autospec=True) as mock:

                client.bootstrap(upload_tools=True, juju_home='temp-home')

            mock.assert_called_with(

                'bootstrap', ('--upload-tools', '--constraints', 'mem=2G'),

                True, juju_home='temp-home')

    def test_bootstrap_async(self):

        env = Environment('foo', '')

        with patch.object(EnvJujuClient, 'juju_async', autospec=True) as mock:

            client = EnvJujuClient(env, None, None)

            with client.bootstrap_async(juju_home='foo'):

                mock.assert_called_once_with(

                    client, 'bootstrap', ('--constraints', 'mem=2G'),

                    juju_home='foo')

    def test_bootstrap_async_upload_tools(self):

        env = Environment('foo', '')

        with patch.object(EnvJujuClient, 'juju_async', autospec=True) as mock:

            client = EnvJujuClient(env, None, None)

            with client.bootstrap_async(upload_tools=True):

                mock.assert_called_with(

                    client, 'bootstrap', ('--upload-tools', '--constraints',

                                          'mem=2G'), juju_home=None)

    def test_destroy_environment_non_sudo(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client.env, 'needs_sudo', lambda: False):

            with patch.object(client, 'juju') as mock:

                client.destroy_environment()

            mock.assert_called_with(

                'destroy-environment', ('foo', '--force', '-y'),

                False, check=False, include_e=False, timeout=600.0)

    def test_destroy_environment_sudo(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client.env, 'needs_sudo', lambda: True):

            with patch.object(client, 'juju') as mock:

                client.destroy_environment()

            mock.assert_called_with(

                'destroy-environment', ('foo', '--force', '-y'),

                True, check=False, include_e=False, timeout=600.0)

    def test_destroy_environment_no_force(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client, 'juju') as mock:

            client.destroy_environment(force=False)

            mock.assert_called_with(

                'destroy-environment', ('foo', '-y'),

                False, check=False, include_e=False, timeout=600.0)

    def test_destroy_environment_delete_jenv(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client, 'juju'):

            with temp_env({}) as juju_home:

                jenv_path = get_jenv_path(juju_home, 'foo')

                os.makedirs(os.path.dirname(jenv_path))

                open(jenv_path, 'w')

                self.assertTrue(os.path.exists(jenv_path))

                client.destroy_environment(delete_jenv=True)

                self.assertFalse(os.path.exists(jenv_path))

    def test_get_juju_output(self):

        env = Environment('foo', '')

        asdf = lambda x, stderr, env: 'asdf'

        client = EnvJujuClient(env, None, None)

        with patch('subprocess.check_output', side_effect=asdf) as mock:

            result = client.get_juju_output('bar')

        self.assertEqual('asdf', result)

        self.assertEqual((('juju', '--show-log', 'bar', '-e', 'foo'),),

                         mock.call_args[0])

    def test_get_juju_output_accepts_varargs(self):

        env = Environment('foo', '')

        asdf = lambda x, stderr, env: 'asdf'

        client = EnvJujuClient(env, None, None)

        with patch('subprocess.check_output', side_effect=asdf) as mock:

            result = client.get_juju_output('bar', 'baz', '--qux')

        self.assertEqual('asdf', result)

        self.assertEqual((('juju', '--show-log', 'bar', '-e', 'foo', 'baz',

                           '--qux'),), mock.call_args[0])

    def test_get_juju_output_stderr(self):

        def raise_without_stderr(args, stderr, env):

            stderr.write('Hello!')

            raise subprocess.CalledProcessError('a', 'b')

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with self.assertRaises(subprocess.CalledProcessError) as exc:

            with patch('subprocess.check_output', raise_without_stderr):

                client.get_juju_output('bar')

        self.assertEqual(exc.exception.stderr, 'Hello!')

    def test_get_juju_output_accepts_timeout(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch('subprocess.check_output') as sco_mock:

            client.get_juju_output('bar', timeout=5)

        self.assertEqual(

            sco_mock.call_args[0][0],

            ('timeout', '5.00s', 'juju', '--show-log', 'bar', '-e', 'foo'))

    def test__shell_environ_cloudsigma(self):

        client = EnvJujuClient(

            SimpleEnvironment('baz', {'type': 'cloudsigma'}),

            '1.24-foobar', 'path')

        env = client._shell_environ()

        self.assertEqual(env.get(JUJU_DEV_FEATURE_FLAGS, ''), '')

    def test_juju_output_supplies_path(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, '/foobar/bar')

        with patch('subprocess.check_output') as sco_mock:

            client.get_juju_output('cmd', 'baz')

        self.assertRegexpMatches(sco_mock.call_args[1]['env']['PATH'],

                                 r'/foobar\:')

    def test_get_status(self):

        output_text = yield dedent("""\

                - a

                - b

                - c

                """)

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        with patch.object(client, 'get_juju_output',

                          return_value=output_text):

            result = client.get_status()

        self.assertEqual(Status, type(result))

        self.assertEqual(['a', 'b', 'c'], result.status)

    def test_get_status_retries_on_error(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        client.attempt = 0

        def get_juju_output(command, *args):

            if client.attempt == 1:

                return '"hello"'

            client.attempt += 1

            raise subprocess.CalledProcessError(1, command)

        with patch.object(client, 'get_juju_output', get_juju_output):

            client.get_status()

    def test_get_status_raises_on_timeout_1(self):

        env = Environment('foo', '')

        client = EnvJujuClient(env, None, None)

        def get_juju_output(command):

            raise subprocess.CalledProcessError(1, command)

        with patch.object(client, 'get_juju_output',

                          side_effect=get_juju_output):

            with patch('jujupy.until_timeout', lambda x: iter([None, None])):

                with self.assertRaisesRegexp(

                        Exception, 'Timed out waiting for juju status'):

                    client.get_status()

    def test_get_status_raises_on_timeout_2(self):

        env = SimpleEnvironment('foo')

        client = EnvJujuClient(env, None, None)

        with patch('jujupy.until_timeout', return_value=iter([1])) as mock_ut:

            with patch.object(client, 'get_juju_output',

                              side_effect=StopIteration):

                with self.assertRaises(StopIteration):

                    client.get_status(500)

        mock_ut.assert_called_with(500)

    @staticmethod

    def make_status_yaml(key, machine_value, unit_value):

        return dedent("""\

            machines:

              "0":

                {0}: {1}

            services:

              jenkins:

                units:

                  jenkins/0:

                    {0}: {2}

        """.format(key, machine_value, unit_value))

    def test_deploy_non_joyent(self):

        env = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'local'}), '1.234-76', None)

        with patch.object(env, 'juju') as mock_juju:

            env.deploy('mondogb')

        mock_juju.assert_called_with('deploy', ('mondogb',))

    def test_deploy_joyent(self):

        env = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'local'}), '1.234-76', None)

        with patch.object(env, 'juju') as mock_juju:

            env.deploy('mondogb')

        mock_juju.assert_called_with('deploy', ('mondogb',))

    def test_deploy_repository(self):

        env = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'local'}), '1.234-76', None)

        with patch.object(env, 'juju') as mock_juju:

            env.deploy('mondogb', '/home/jrandom/repo')

        mock_juju.assert_called_with(

            'deploy', ('mondogb', '--repository', '/home/jrandom/repo'))

    def test_status_until_always_runs_once(self):

        client = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'local'}), '1.234-76', None)

        status_txt = self.make_status_yaml('agent-state', 'started', 'started')

        with patch.object(client, 'get_juju_output', return_value=status_txt):

            result = list(client.status_until(-1))

        self.assertEqual(

            [r.status for r in result], [Status.from_text(status_txt).status])

    def test_status_until_timeout(self):

        client = EnvJujuClient(

            SimpleEnvironment('foo', {'type': 'local'}), '1.234-76', None)

        status_txt = self.make_status_yaml('agent-state', 'started', 'started')

        status_yaml = yaml.safe_load(status_txt)

        def until_timeout_stub(timeout, start=None):

            return iter([None, None])

        with patch.object(client, 'get_juju_output', return_value=status_txt):

            with patch('jujupy.until_timeout',

                       side_effect=until_timeout_stub) as ut_mock:

                result = list(client.status_until(30, 70))

        self.assertEqual(

            [r.status for r in result], [status_yaml] * 3)

        # until_timeout is called by status as well as status_until.

        self.assertEqual(ut_mock.mock_calls,

                         [call(60), call(30, start=70), call(60), call(60)])

    def test_wait_for_started(self):

        value = self.make_status_yaml('agent-state', 'started', 'started')

        client = EnvJujuClient(SimpleEnvironment('local'), None, None)

        with patch.object(client, 'get_juju_output', return_value=value):

            client.wait_for_started()

    def test_wait_for_started_timeout(self):

        value = self.make_status_yaml('agent-state', 'pending', 'started')