~andrewjbeach/juju-ci-tools/make-local-patcher

« back to all changes in this revision

Viewing changes to tests/test_chaos.py

  • Committer: Aaron Bentley
  • Date: 2015-01-19 15:32:33 UTC
  • mto: This revision was merged to the branch mainline in revision 804.
  • Revision ID: aaron.bentley@canonical.com-20150119153233-jjcvikwiw1dx2lak
Print error on missing environment.

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
import os
2
 
from shutil import rmtree
3
 
import stat
4
 
from tempfile import (
5
 
    NamedTemporaryFile,
6
 
    mkdtemp,
7
 
    )
8
 
import unittest
9
 
 
10
 
from mock import (
11
 
    call,
12
 
    patch,
13
 
)
14
 
import yaml
15
 
 
16
 
from chaos import (
17
 
    background_chaos,
18
 
    MonkeyRunner,
19
 
    )
20
 
from jujupy import (
21
 
    EnvJujuClient,
22
 
    JujuData,
23
 
    )
24
 
from remote import SSHRemote
25
 
from tests import FakeHomeTestCase
26
 
from test_jujupy import (
27
 
    assert_juju_call,
28
 
    )
29
 
 
30
 
 
31
 
class TestBackgroundChaos(FakeHomeTestCase):
32
 
 
33
 
    def test_background_chaos(self):
34
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
35
 
        with patch('chaos.MonkeyRunner.deploy_chaos_monkey',
36
 
                   autospec=True) as d_mock:
37
 
            with patch('chaos.MonkeyRunner.unleash_once',
38
 
                       return_value=['1'], autospec=True) as u_mock:
39
 
                with patch('chaos.MonkeyRunner.wait_for_chaos',
40
 
                           autospec=True) as w_mock:
41
 
                    with patch('chaos.remote_from_unit',
42
 
                               return_value=SSHRemote(None, None, 'foo')
43
 
                               ) as ru_mock:
44
 
                        with patch('remote.SSHRemote.copy') as rc_mock:
45
 
                            log_dir = mkdtemp()
46
 
                            with background_chaos('foo', client, log_dir, 1):
47
 
                                pass
48
 
                            rmtree(log_dir)
49
 
        self.assertEqual(1, d_mock.call_count)
50
 
        self.assertEqual(1, u_mock.call_count)
51
 
        self.assertEqual(1, ru_mock.call_count)
52
 
        self.assertEqual(1, rc_mock.call_count)
53
 
        self.assertEqual({'state': 'start'}, w_mock.mock_calls[0][2])
54
 
        self.assertEqual({'state': 'complete', 'timeout': 1},
55
 
                         w_mock.mock_calls[1][2])
56
 
 
57
 
    def test_background_chaos_exits(self):
58
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
59
 
        with patch('chaos.MonkeyRunner.deploy_chaos_monkey',
60
 
                   autospec=True):
61
 
            with patch('chaos.MonkeyRunner.unleash_once',
62
 
                       autospec=True):
63
 
                with patch('chaos.MonkeyRunner.wait_for_chaos',
64
 
                           autospec=True):
65
 
                    with patch('chaos.remote_from_unit'):
66
 
                        with patch('remote.SSHRemote.copy'):
67
 
                            with patch('logging.exception') as le_mock:
68
 
                                with patch('sys.exit', autospec=True) as sm:
69
 
                                    log_dir = mkdtemp()
70
 
                                    with background_chaos('foo', client,
71
 
                                                          log_dir, 1):
72
 
                                        raise Exception()
73
 
                            rmtree(log_dir)
74
 
        self.assertEqual(1, le_mock.call_count)
75
 
        sm.assert_called_once_with(1)
76
 
 
77
 
 
78
 
class TestRunChaosMonkey(FakeHomeTestCase):
79
 
 
80
 
    def test_deploy_chaos_monkey(self):
81
 
        def output(*args, **kwargs):
82
 
            status = yaml.safe_dump({
83
 
                'machines': {
84
 
                    '0': {'agent-state': 'started'}
85
 
                },
86
 
                'applications': {
87
 
                    'ser1': {
88
 
                        'units': {
89
 
                            'bar': {
90
 
                                'agent-state': 'started',
91
 
                                'subordinates': {
92
 
                                    'chaos-monkey/1': {
93
 
                                        'agent-state': 'started'
94
 
                                    }
95
 
                                }
96
 
                            }
97
 
                        }
98
 
                    }
99
 
                }
100
 
            })
101
 
            output = {
102
 
                ('show-status', '--format', 'yaml'): status,
103
 
                }
104
 
            return output[args]
105
 
        client = EnvJujuClient(JujuData('foo', {}), '1.25.0', '/foo/juju')
106
 
        with patch.object(client, 'get_juju_output', side_effect=output,
107
 
                          autospec=True) as gjo_mock:
108
 
            with patch('subprocess.check_call', autospec=True) as cc_mock:
109
 
                monkey_runner = MonkeyRunner('foo', client, service='ser1')
110
 
                with patch('jujupy.GroupReporter._write', autospec=True):
111
 
                    monkey_runner.deploy_chaos_monkey()
112
 
        assert_juju_call(
113
 
            self,
114
 
            cc_mock,
115
 
            client,
116
 
            ('juju', '--show-log', 'deploy',
117
 
             '-m', 'foo:foo', 'local:chaos-monkey'),
118
 
            0)
119
 
        assert_juju_call(self, cc_mock, client, (
120
 
            'juju', '--show-log', 'add-relation', '-m', 'foo:foo', 'ser1',
121
 
            'chaos-monkey'), 1)
122
 
        self.assertEqual(cc_mock.call_count, 2)
123
 
        self.assertEqual(gjo_mock.call_count, 2)
124
 
 
125
 
    def test_iter_chaos_monkey_units(self):
126
 
        def output(*args, **kwargs):
127
 
            status = yaml.safe_dump({
128
 
                'machines': {
129
 
                    '0': {'agent-state': 'started'}
130
 
                },
131
 
                'applications': {
132
 
                    'jenkins': {
133
 
                        'units': {
134
 
                            'foo': {
135
 
                                'subordinates': {
136
 
                                    'chaos-monkey/0': {'baz': 'qux'},
137
 
                                    'not-chaos/0': {'qwe': 'rty'},
138
 
                                }
139
 
                            },
140
 
                            'bar': {
141
 
                                'subordinates': {
142
 
                                    'chaos-monkey/1': {'abc': '123'},
143
 
                                }
144
 
                            }
145
 
                        }
146
 
                    }
147
 
                }
148
 
            })
149
 
            output = {
150
 
                ('show-status', '--format', 'yaml'): status,
151
 
                }
152
 
            return output[args]
153
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
154
 
        runner = MonkeyRunner('foo', client, service='jenkins')
155
 
        with patch.object(client, 'get_juju_output', side_effect=output,
156
 
                          autospec=True):
157
 
            monkey_units = dict((k, v) for k, v in
158
 
                                runner.iter_chaos_monkey_units())
159
 
        expected = {
160
 
            'chaos-monkey/0': {'baz': 'qux'},
161
 
            'chaos-monkey/1': {'abc': '123'}
162
 
        }
163
 
        self.assertEqual(expected, monkey_units)
164
 
 
165
 
    def test_get_unit_status(self):
166
 
        def output(*args, **kwargs):
167
 
            status = yaml.safe_dump({
168
 
                'machines': {
169
 
                    '0': {'agent-state': 'started'}
170
 
                },
171
 
                'applications': {
172
 
                    'jenkins': {
173
 
                        'units': {
174
 
                            'foo': {
175
 
                                'subordinates': {
176
 
                                    'chaos-monkey/0': {'baz': 'qux'},
177
 
                                    'not-chaos/0': {'qwe': 'rty'},
178
 
                                }
179
 
                            },
180
 
                            'bar': {
181
 
                                'subordinates': {
182
 
                                    'chaos-monkey/1': {'abc': '123'},
183
 
                                }
184
 
                            }
185
 
                        }
186
 
                    }
187
 
                }
188
 
            })
189
 
            charm_config = yaml.safe_dump({
190
 
                'charm': {'jenkins'},
191
 
                'service': {'jenkins'},
192
 
                'settings': {
193
 
                    'chaos-dir': {
194
 
                        'default': 'true',
195
 
                        'description': 'bla bla',
196
 
                        'type': 'string',
197
 
                        'value': '/tmp/charm-dir',
198
 
                    }
199
 
                }
200
 
            })
201
 
            output = {
202
 
                ('status',): status,
203
 
                ('config', 'chaos-monkey'): charm_config,
204
 
                }
205
 
            return output[args]
206
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo')
207
 
        monkey_runner = MonkeyRunner('foo', client, service='jenkins')
208
 
        monkey_runner.monkey_ids = {
209
 
            'chaos-monkey/0': 'workspace0',
210
 
            'chaos-monkey/1': 'workspace1'
211
 
        }
212
 
        with patch.object(client, 'get_juju_output', side_effect=output,
213
 
                          autospec=True):
214
 
            with patch('subprocess.call', autospec=True,
215
 
                       return_value=0) as call_mock:
216
 
                for unit_name in ['chaos-monkey/1', 'chaos-monkey/0']:
217
 
                    self.assertEqual(monkey_runner.get_unit_status(unit_name),
218
 
                                     'running')
219
 
            self.assertEqual(call_mock.call_count, 2)
220
 
        with patch.object(client, 'get_juju_output', side_effect=output,
221
 
                          autospec=True):
222
 
            with patch('subprocess.call', autospec=True,
223
 
                       return_value=1) as call_mock:
224
 
                for unit_name in ['chaos-monkey/1', 'chaos-monkey/0']:
225
 
                    self.assertEqual(monkey_runner.get_unit_status(unit_name),
226
 
                                     'done')
227
 
            self.assertEqual(call_mock.call_count, 2)
228
 
 
229
 
 
230
 
class TestUnleashOnce(FakeHomeTestCase):
231
 
 
232
 
    def test_unleash_once(self):
233
 
        def output(*args, **kwargs):
234
 
            status = yaml.safe_dump({
235
 
                'machines': {
236
 
                    '0': {'agent-state': 'started'}
237
 
                },
238
 
                'applications': {
239
 
                    'jenkins': {
240
 
                        'units': {
241
 
                            'foo': {
242
 
                                'subordinates': {
243
 
                                    'chaos-monkey/0': {'baz': 'qux'},
244
 
                                    'not-chaos/0': {'qwe': 'rty'},
245
 
                                }
246
 
                            },
247
 
                            'bar': {
248
 
                                'subordinates': {
249
 
                                    'chaos-monkey/1': {'abc': '123'},
250
 
                                }
251
 
                            }
252
 
                        }
253
 
                    }
254
 
                }
255
 
            })
256
 
            charm_config = yaml.safe_dump({
257
 
                'charm': {'jenkins'},
258
 
                'service': {'jenkins'},
259
 
                'settings': {
260
 
                    'chaos-dir': {
261
 
                        'default': 'true',
262
 
                        'description': 'bla bla',
263
 
                        'type': 'string',
264
 
                        'value': '/tmp/charm-dir',
265
 
                    }
266
 
                }
267
 
            })
268
 
            output = {
269
 
                ('show-status', '--format', 'yaml'): status,
270
 
                ('get', 'jenkins'): charm_config,
271
 
                ('run-action', 'chaos-monkey/0', 'start', 'mode=single',
272
 
                 'enablement-timeout=120'
273
 
                 ): ('Action queued with id: '
274
 
                     'abcdabcdabcdabcdabcdabcdabcdabcdabcd'),
275
 
                ('run-action', 'chaos-monkey/0', 'start', 'mode=single',
276
 
                 'enablement-timeout=120',
277
 
                 'monkey-id=abcdabcdabcdabcdabcdabcdabcdabcdabcd'
278
 
                 ): ('Action queued with id: '
279
 
                     'efabefabefabefabefabefabefabefabefab'),
280
 
                ('run-action', 'chaos-monkey/1', 'start', 'mode=single',
281
 
                 'enablement-timeout=120'
282
 
                 ): ('Action queued with id: '
283
 
                     '123412341234123412341234123412341234'),
284
 
                ('run-action', 'chaos-monkey/1', 'start', 'mode=single',
285
 
                 'enablement-timeout=120',
286
 
                 'monkey-id=123412341234123412341234123412341234'
287
 
                 ): ('Action queued with id: '
288
 
                     '567856785678567856785678567856785678'),
289
 
                }
290
 
            return output[args]
291
 
 
292
 
        def output2(*args, **kwargs):
293
 
            status = yaml.safe_dump({
294
 
                'machines': {
295
 
                    '0': {'agent-state': 'started'}
296
 
                },
297
 
                'applications': {
298
 
                    'jenkins': {
299
 
                        'units': {
300
 
                            'bar': {
301
 
                                'subordinates': {
302
 
                                    'chaos-monkey/1': {'abc': '123'},
303
 
                                }
304
 
                            }
305
 
                        }
306
 
                    }
307
 
                }
308
 
            })
309
 
            output = {
310
 
                ('show-status', '--format', 'yaml'): status,
311
 
                ('run-action', 'chaos-monkey/1', 'start', 'mode=single',
312
 
                 'enablement-timeout=120',
313
 
                 'monkey-id=123412341234123412341234123412341234'
314
 
                 ): ('Action queued with id: '
315
 
                     'abcdabcdabcdabcdabcdabcdabcdabcdabcd'),
316
 
                }
317
 
            return output[args]
318
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
319
 
        monkey_runner = MonkeyRunner('foo', client, service='jenkins')
320
 
        with patch.object(client, 'get_juju_output', side_effect=output,
321
 
                          autospec=True) as gjo_mock:
322
 
            returned = monkey_runner.unleash_once()
323
 
        expected = ['abcd' * 9, '1234' * 9]
324
 
        self.assertEqual(
325
 
            [
326
 
                call('show-status', '--format', 'yaml', controller=False),
327
 
                call('run-action', 'chaos-monkey/1', 'start', 'mode=single',
328
 
                     'enablement-timeout=120'),
329
 
                call('run-action', 'chaos-monkey/0', 'start', 'mode=single',
330
 
                     'enablement-timeout=120'),
331
 
            ],
332
 
            gjo_mock.call_args_list)
333
 
        self.assertEqual(['chaos-monkey/1', 'chaos-monkey/0'],
334
 
                         monkey_runner.monkey_ids.keys())
335
 
        self.assertEqual(len(monkey_runner.monkey_ids), 2)
336
 
        self.assertItemsEqual(returned, expected)
337
 
        with patch.object(client, 'get_juju_output', side_effect=output,
338
 
                          autospec=True) as gjo_mock:
339
 
            monkey_runner.unleash_once()
340
 
        self.assertEqual(
341
 
            [
342
 
                call('show-status', '--format', 'yaml', controller=False),
343
 
                call('run-action',
344
 
                     'chaos-monkey/1', 'start', 'mode=single',
345
 
                     'enablement-timeout=120',
346
 
                     'monkey-id=123412341234123412341234123412341234'),
347
 
                call('run-action',
348
 
                     'chaos-monkey/0', 'start', 'mode=single',
349
 
                     'enablement-timeout=120',
350
 
                     'monkey-id=abcdabcdabcdabcdabcdabcdabcdabcdabcd'),
351
 
            ],
352
 
            gjo_mock.call_args_list)
353
 
        self.assertTrue('1234', monkey_runner.monkey_ids['chaos-monkey/1'])
354
 
        # Test monkey_ids.get(unit_name) does not change on second call to
355
 
        # unleash_once()
356
 
        with patch.object(client, 'get_juju_output', side_effect=output2,
357
 
                          autospec=True):
358
 
            monkey_runner.unleash_once()
359
 
        self.assertEqual('1234' * 9,
360
 
                         monkey_runner.monkey_ids['chaos-monkey/1'])
361
 
 
362
 
    def test_unleash_once_raises_for_unexpected_action_output(self):
363
 
        def output(*args, **kwargs):
364
 
            status = yaml.safe_dump({
365
 
                'machines': {
366
 
                    '0': {'agent-state': 'started'}
367
 
                },
368
 
                'applications': {
369
 
                    'jenkins': {
370
 
                        'units': {
371
 
                            'foo': {
372
 
                                'subordinates': {
373
 
                                    'chaos-monkey/0': {'baz': 'qux'},
374
 
                                }
375
 
                            }
376
 
                        }
377
 
                    }
378
 
                }
379
 
            })
380
 
            output = {
381
 
                ('show-status', '--format', 'yaml'): status,
382
 
                ('run-action', 'chaos-monkey/0', 'start', 'mode=single',
383
 
                 'enablement-timeout=120'
384
 
                 ): 'Action fail',
385
 
                }
386
 
            return output[args]
387
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
388
 
        monkey_runner = MonkeyRunner('foo', client, service='jenkins')
389
 
        with patch.object(client, 'get_juju_output', side_effect=output,
390
 
                          autospec=True):
391
 
            with self.assertRaisesRegexp(
392
 
                    Exception, 'Action id not found in output: Action fail'):
393
 
                monkey_runner.unleash_once()
394
 
 
395
 
 
396
 
class TestIsHealthy(unittest.TestCase):
397
 
 
398
 
    def test_is_healthy(self):
399
 
        SCRIPT = """#!/bin/bash\necho -n 'PASS'\nexit 0"""
400
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
401
 
        with NamedTemporaryFile(delete=False) as health_script:
402
 
            health_script.write(SCRIPT)
403
 
            os.fchmod(health_script.fileno(), stat.S_IEXEC | stat.S_IREAD)
404
 
            health_script.close()
405
 
            monkey_runner = MonkeyRunner('foo', client,
406
 
                                         health_checker=health_script.name)
407
 
            with patch('logging.info') as lo_mock:
408
 
                result = monkey_runner.is_healthy()
409
 
            os.unlink(health_script.name)
410
 
            self.assertTrue(result)
411
 
            self.assertEqual(lo_mock.call_args[0][0],
412
 
                             'Health check output: PASS')
413
 
 
414
 
    def test_is_healthy_fail(self):
415
 
        SCRIPT = """#!/bin/bash\necho -n 'FAIL'\nexit 1"""
416
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
417
 
        with NamedTemporaryFile(delete=False) as health_script:
418
 
            health_script.write(SCRIPT)
419
 
            os.fchmod(health_script.fileno(), stat.S_IEXEC | stat.S_IREAD)
420
 
            health_script.close()
421
 
            monkey_runner = MonkeyRunner('foo', client,
422
 
                                         health_checker=health_script.name)
423
 
            with patch('logging.error') as le_mock:
424
 
                result = monkey_runner.is_healthy()
425
 
            os.unlink(health_script.name)
426
 
            self.assertFalse(result)
427
 
            self.assertEqual(le_mock.call_args[0][0], 'FAIL')
428
 
 
429
 
    def test_is_healthy_with_no_execute_perms(self):
430
 
        SCRIPT = """#!/bin/bash\nexit 0"""
431
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo/juju')
432
 
        with NamedTemporaryFile(delete=False) as health_script:
433
 
            health_script.write(SCRIPT)
434
 
            os.fchmod(health_script.fileno(), stat.S_IREAD)
435
 
            health_script.close()
436
 
            monkey_runner = MonkeyRunner('foo', client,
437
 
                                         health_checker=health_script.name)
438
 
            with patch('logging.error') as le_mock:
439
 
                with self.assertRaises(OSError):
440
 
                    monkey_runner.is_healthy()
441
 
            os.unlink(health_script.name)
442
 
        self.assertRegexpMatches(
443
 
            le_mock.call_args[0][0],
444
 
            r'The health check failed to execute with: \[Errno 13\].*')
445
 
 
446
 
 
447
 
class TestWaitForChaos(FakeHomeTestCase):
448
 
 
449
 
    def test_wait_for_chaos_complete(self):
450
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo')
451
 
        runner = MonkeyRunner('foo', client)
452
 
        units = [('blib', 'blab')]
453
 
        with patch.object(runner, 'iter_chaos_monkey_units', autospec=True,
454
 
                          return_value=units) as ic_mock:
455
 
            with patch.object(runner, 'get_unit_status',
456
 
                              autospec=True, return_value='done') as us_mock:
457
 
                returned = runner.wait_for_chaos()
458
 
        self.assertEqual(returned, None)
459
 
        self.assertEqual(ic_mock.call_count, 1)
460
 
        self.assertEqual(us_mock.call_count, 1)
461
 
 
462
 
    def test_wait_for_chaos_complete_timesout(self):
463
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo')
464
 
        runner = MonkeyRunner('foo', client)
465
 
        with self.assertRaisesRegexp(
466
 
                Exception, 'Chaos operations did not complete.'):
467
 
            runner.wait_for_chaos(timeout=0)
468
 
 
469
 
    def test_wait_for_chaos_started(self):
470
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo')
471
 
        runner = MonkeyRunner('foo', client)
472
 
        units = [('blib', 'blab')]
473
 
        with patch.object(runner, 'iter_chaos_monkey_units', autospec=True,
474
 
                          return_value=units) as ic_mock:
475
 
            with patch.object(runner, 'get_unit_status',
476
 
                              autospec=True,
477
 
                              return_value='running') as us_mock:
478
 
                returned = runner.wait_for_chaos(state='start')
479
 
        self.assertEqual(returned, None)
480
 
        self.assertEqual(ic_mock.call_count, 1)
481
 
        self.assertEqual(us_mock.call_count, 1)
482
 
 
483
 
    def test_wait_for_chaos_unexpected_state(self):
484
 
        client = EnvJujuClient(JujuData('foo', {}), None, '/foo')
485
 
        runner = MonkeyRunner('foo', client)
486
 
        with self.assertRaisesRegexp(
487
 
                Exception, 'Unexpected state value: foo'):
488
 
            runner.wait_for_chaos(state='foo')