69
66
def terminate(env, instance_ids):
70
model = client._backend.controller_state.controller_model
71
67
for instance_id in instance_ids:
72
model.remove_state_server(instance_id)
68
admin_model = client._backend.controller_state.admin_model
69
admin_model.remove_state_server(instance_id)
74
71
with patch('assess_recovery.wait_for_state_server_to_shutdown',
76
73
with patch('assess_recovery.terminate_instances',
77
74
side_effect=terminate):
78
75
with patch('deploy_stack.wait_for_port', autospec=True):
79
with patch('assess_recovery.restore_present_state_server',
81
with patch('assess_recovery.check_token',
83
side_effect=['Token: One', 'Token: Two']):
84
with patch('assess_recovery.show_controller',
86
return_value='controller'):
89
78
def test_backup(self):
90
79
client = fake_juju_client()
104
93
with self.assess_recovery_cxt(client):
105
94
assess_recovery(bs_manager, 'ha-backup', 'trusty')
107
def test_controller_model_backup(self):
96
def test_admin_model_backup(self):
108
97
client = fake_juju_client()
109
98
bs_manager = Mock(client=client, known_hosts={})
110
99
with self.assess_recovery_cxt(client):
111
100
assess_recovery(bs_manager, 'backup', 'trusty')
113
def test_controller_model_ha(self):
102
def test_admin_model_ha(self):
114
103
client = fake_juju_client()
115
104
bs_manager = Mock(client=client, known_hosts={})
116
105
with self.assess_recovery_cxt(client):
117
106
assess_recovery(bs_manager, 'ha', 'trusty')
119
def test_controller_model_ha_backup(self):
108
def test_admin_model_ha_backup(self):
120
109
client = fake_juju_client()
121
110
bs_manager = Mock(client=client, known_hosts={})
122
111
with self.assess_recovery_cxt(client):
126
115
@patch('assess_recovery.configure_logging', autospec=True)
127
116
@patch('assess_recovery.BootstrapManager.booted_context', autospec=True)
117
@patch('jujupy.SimpleEnvironment.from_config', return_value=sentinel.env)
128
118
class TestMain(FakeHomeTestCase):
130
def test_main(self, mock_bc, mock_cl):
120
def test_main(self, mock_e, mock_bc, mock_cl):
131
121
client = Mock(spec=['is_jes_enabled', 'version'])
132
122
client.version = '1.25.5'
133
with patch('deploy_stack.client_from_config',
123
with patch('jujupy.EnvJujuClient.by_version',
134
124
return_value=client) as mock_c:
135
125
with patch('assess_recovery.assess_recovery',
136
126
autospec=True) as mock_assess:
137
127
main(['an-env', '/juju', 'log_dir', 'tmp-env', '--backup',
138
128
'--charm-series', 'a-series'])
139
129
mock_cl.assert_called_once_with(logging.INFO)
140
mock_c.assert_called_once_with('an-env', '/juju', debug=False,
130
mock_e.assert_called_once_with('an-env')
131
mock_c.assert_called_once_with(sentinel.env, '/juju', debug=False)
142
132
self.assertEqual(mock_bc.call_count, 1)
143
133
self.assertEqual(mock_assess.call_count, 1)
144
134
bs_manager, strategy, series = mock_assess.call_args[0]
145
135
self.assertEqual((bs_manager.client, strategy, series),
146
136
(client, 'backup', 'a-series'))
148
def test_error(self, mock_bc, mock_cl):
138
def test_error(self, mock_e, mock_bc, mock_cl):
149
139
class FakeError(Exception):
150
140
"""Custom exception to validate error handling."""
151
141
error = FakeError('An error during test')
152
142
client = Mock(spec=['is_jes_enabled', 'version'])
153
143
client.version = '2.0.0'
154
with patch('deploy_stack.client_from_config',
144
with patch('jujupy.EnvJujuClient.by_version',
155
145
return_value=client) as mock_c:
156
146
with patch('assess_recovery.parse_new_state_server_from_error',
157
147
autospec=True, return_value='a-host') as mock_pe:
205
194
call(client.env, ['juju-dddd-machine-3'])],
206
195
ti_mock.mock_calls)
207
196
self.assertEqual(
208
[call('10.0.0.2', client, 'juju-cccc-machine-2', timeout=120),
209
call('10.0.0.0', client, 'juju-aaaa-machine-0', timeout=120),
210
call('10.0.0.3', client, 'juju-dddd-machine-3', timeout=120)],
197
[call('10.0.0.2', client, 'juju-cccc-machine-2'),
198
call('10.0.0.0', client, 'juju-aaaa-machine-0'),
199
call('10.0.0.3', client, 'juju-dddd-machine-3')],
211
200
wsss_mock.mock_calls)
212
201
self.assertEqual(
213
202
self.log_stream.getvalue(),
231
219
client.get_controller_leader.assert_called_once_with()
232
220
ti_mock.assert_called_once_with(client.env, ['juju-dddd-machine-3'])
233
221
wsss_mock.assert_called_once_with(
234
'10.0.0.3', client, 'juju-dddd-machine-3', timeout=120)
222
'10.0.0.3', client, 'juju-dddd-machine-3')
235
223
self.assertEqual(
236
224
self.log_stream.getvalue(),
237
225
'INFO Instrumenting node failure for member 3:'
238
226
' juju-dddd-machine-3 at 10.0.0.3\n')
240
def test_delete_controller_members_azure(self, ti_mock, wsss_mock):
241
client = Mock(spec=['env', 'get_controller_leader'])
242
client.env = sentinel.env
243
client.env.config = {'type': 'azure'}
244
client.get_controller_leader.return_value = Machine('3', {
245
'dns-name': '10.0.0.3',
246
'instance-id': 'juju-dddd-machine-3',
247
'controller-member-status': 'has-vote'})
248
with patch('assess_recovery.convert_to_azure_ids', autospec=True,
249
return_value=['juju-azure-id']):
250
deleted = delete_controller_members(client, leader_only=True)
251
self.assertEqual(['3'], deleted)
252
client.get_controller_leader.assert_called_once_with()
253
ti_mock.assert_called_once_with(client.env, ['juju-azure-id'])
254
wsss_mock.assert_called_once_with(
255
'10.0.0.3', client, 'juju-azure-id', timeout=120)
257
self.log_stream.getvalue(),
258
'INFO Instrumenting node failure for member 3:'
259
' juju-azure-id at 10.0.0.3\n')
262
class TestRestoreMissingStateServer(FakeHomeTestCase):
264
def test_restore_missing_state_server_with_check_controller(self):
265
client = Mock(spec=['env', 'set_config', 'wait_for_started',
266
'wait_for_workloads'])
267
controller_client = Mock(spec=['restore_backup', 'wait_for_started'])
268
with patch('assess_recovery.check_token',
269
autospec=True, return_value='Token: Two'):
270
with patch('assess_recovery.show_controller', autospec=True):
271
restore_missing_state_server(
272
client, controller_client, 'backup_file',
273
check_controller=True)
274
controller_client.restore_backup.assert_called_once_with('backup_file')
275
controller_client.wait_for_started.assert_called_once_with(600)
276
client.set_config.assert_called_once_with(
277
'dummy-source', {'token': 'Two'})
278
client.wait_for_started.assert_called_once_with()
279
client.wait_for_workloads.assert_called_once_with()
281
def test_restore_missing_state_server_without_check_controller(self):
282
client = Mock(spec=['env', 'set_config', 'wait_for_started',
283
'wait_for_workloads'])
284
controller_client = Mock(spec=['restore_backup', 'wait_for_started'])
285
with patch('assess_recovery.check_token',
286
autospec=True, return_value='Token: Two'):
287
with patch('assess_recovery.show_controller', autospec=True):
288
restore_missing_state_server(
289
client, controller_client, 'backup_file',
290
check_controller=False)
291
self.assertEqual(0, controller_client.wait_for_started.call_count)
294
class TestCheckToken(TestCase):
296
def test_check_token_found(self):
298
with patch('assess_recovery.get_token_from_status', autospec=True,
299
side_effect=['Token: foo']):
300
found = check_token(client, 'foo')
301
self.assertEqual('Token: foo', found)
303
def test_check_token_none_before_found(self):
305
with patch('assess_recovery.get_token_from_status', autospec=True,
306
side_effect=[None, 'foo']):
307
found = check_token(client, 'foo')
308
self.assertEqual('foo', found)
310
def test_check_token_other_before_found(self):
312
with patch('assess_recovery.get_token_from_status', autospec=True,
313
side_effect=['Starting', 'foo']):
314
found = check_token(client, 'foo')
315
self.assertEqual('foo', found)
317
def test_check_token_not_found(self):
319
with patch('assess_recovery.get_token_from_status', autospec=True,
320
return_value='other'):
321
with patch('assess_recovery.until_timeout', autospec=True,
322
side_effect=['1', '0']):
323
with self.assertRaises(JujuAssertionError):
324
check_token(client, 'foo')