← Back to branch summary

~martin-nowack/ubuntu/utopic/maas/bug-1425837

« back to all changes in this revision

Viewing changes to src/provisioningserver/rpc/tests/test_power.py

  • Committer: Package Import Robot
  • Author(s): Andres Rodriguez, Diogo Matsubara, Jeroen Vermeulen, Raphaël Badin, Greg Lutostanski, Gavin Panella, Julian Edwards, Graham Binns, Andres Rodriguez
  • Date: 2014-08-21 14:05:40 UTC
  • mfrom: (1.2.35)
  • Revision ID: package-import@ubuntu.com-20140821140540-khodrzopm91kl4le
Tags: 1.7.0~beta1+bzr2781-0ubuntu1
https://launchpad.net/bugs/1328659
https://launchpad.net/bugs/1334401
https://launchpad.net/bugs/1352273
https://launchpad.net/bugs/1332532
https://launchpad.net/bugs/1300507
* New upstream release, 1.7.0 Beta 1

[Diogo Matsubara]
* debian/control:
  - maas-cluster-controller depends on syslinux-dev | 
    syslinux-common (LP: #1328659)
  - python-maas-provisioningserver depends on
    python-paramiko (LP: #1334401)

[Jeroen Vermeulen]
* debian/extras/99-maas-sudoers:
  - Let maas user import, including sudo tgt-admin and sudo uec2roottar.
* debian/maas-cluster-controller.install:
  - Stop installing obsolete file bootresources.yaml.

[ Raphaël Badin ]
* debian/control:
  - maas-cluster-controller depends on python-pexpect
* debian/extras/99-maas-sudoers:
  - Add rule 'maas-dhcp-server stop' job.

[ Greg Lutostanski ]
* debian/control:
  - maas-cluster-controller depends on grub-common
  - maas-provisioningserver not maas-cluster-controller depends on
    python-pexpect (LP: #1352273)
  - maas-provisioningserver not maas-cluster-controller depends on
    python-seamicroclient (LP: #1332532)

[ Gavin Panella ]
* debian/maas-cluster-controller.postinst
  - Allow maas-pserv to bind to all IPv6 addresses too.

[ Julian Edwards ]
* debian/maas-region-controller-min.apport
  debian/maas-region-controller-min.logrotate
  debian/maas-region-controller-min.postinst
  debian/maas-region-controller.postinst
  - Change the log file name maas.log to maas-django.log
* debian/maas-cluster-controller.postinst
  debian/maas-common.install
  debian/maas-region-controller-min.postinst
  debian/maas-region-controller.postinst
  - Install /var/log/maas/maas.log as a syslog file.
  - Ensure logging is set up for upgrades 

[ Graham Binns ]
* debian/maas-region-controller.postinst:
  - Add symlinks for squid3, squid-deb-proxy and apache log directories to
    /var/log/maas.

[ Andres Rodriguez ]
* debian/maas-region-controller.postinst: Force symlink creation
  for external logs.
* debian/maas-region-controller.postinst: Do not change celery's
  rabbitmq password on upgrade that to not lock remote
  Cluster Controllers if upgrading from 1.5+. (LP: #1300507)

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/provisioningserver/rpc/tests/test_power.py
 
1
# Copyright 2014 Canonical Ltd.  This software is licensed under the
 
2
# GNU Affero General Public License version 3 (see the file LICENSE).
 
3
 
 
4
"""Tests for :py:module:`~provisioningserver.rpc.power`."""
 
5
 
 
6
from __future__ import (
 
7
    absolute_import,
 
8
    print_function,
 
9
    unicode_literals,
 
10
    )
 
11
 
 
12
str = None
 
13
 
 
14
__metaclass__ = type
 
15
__all__ = []
 
16
 
 
17
 
 
18
import random
 
19
 
 
20
from maastesting.factory import factory
 
21
from maastesting.matchers import (
 
22
    MockCalledOnceWith,
 
23
    MockCallsMatch,
 
24
    MockNotCalled,
 
25
    )
 
26
from maastesting.testcase import MAASTestCase
 
27
from mock import (
 
28
    ANY,
 
29
    call,
 
30
    Mock,
 
31
    )
 
32
import provisioningserver
 
33
from provisioningserver.events import EVENT_TYPES
 
34
from provisioningserver.power.poweraction import PowerActionFail
 
35
from provisioningserver.rpc import (
 
36
    power,
 
37
    region,
 
38
    )
 
39
from provisioningserver.rpc.testing import MockClusterToRegionRPCFixture
 
40
from testtools.deferredruntest import (
 
41
    assert_fails_with,
 
42
    AsynchronousDeferredRunTest,
 
43
    )
 
44
from twisted.internet.defer import maybeDeferred
 
45
from twisted.internet.task import Clock
 
46
 
 
47
 
 
48
class TestPowerHelpers(MAASTestCase):
 
49
 
 
50
    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
 
51
 
 
52
    def patch_rpc_methods(self):
 
53
        fixture = self.useFixture(MockClusterToRegionRPCFixture())
 
54
        protocol, io = fixture.makeEventLoop(
 
55
            region.MarkNodeBroken, region.UpdateNodePowerState,
 
56
            region.SendEvent)
 
57
        return protocol, io
 
58
 
 
59
    def test_power_change_success_emits_event(self):
 
60
        system_id = factory.make_name('system_id')
 
61
        hostname = factory.make_name('hostname')
 
62
        power_change = 'on'
 
63
        protocol, io = self.patch_rpc_methods()
 
64
        d = power.power_change_success(system_id, hostname, power_change)
 
65
        io.flush()
 
66
        self.assertThat(
 
67
            protocol.UpdateNodePowerState,
 
68
            MockCalledOnceWith(
 
69
                ANY,
 
70
                system_id=system_id,
 
71
                power_state=power_change)
 
72
        )
 
73
        self.assertThat(
 
74
            protocol.SendEvent,
 
75
            MockCalledOnceWith(
 
76
                ANY,
 
77
                type_name=EVENT_TYPES.NODE_POWERED_ON,
 
78
                system_id=system_id,
 
79
                description='')
 
80
        )
 
81
        return d
 
82
 
 
83
    def test_power_change_starting_emits_event(self):
 
84
        system_id = factory.make_name('system_id')
 
85
        hostname = factory.make_name('hostname')
 
86
        power_change = 'on'
 
87
        protocol, io = self.patch_rpc_methods()
 
88
        d = power.power_change_starting(system_id, hostname, power_change)
 
89
        io.flush()
 
90
        self.assertThat(
 
91
            protocol.SendEvent,
 
92
            MockCalledOnceWith(
 
93
                ANY,
 
94
                type_name=EVENT_TYPES.NODE_POWER_ON_STARTING,
 
95
                system_id=system_id,
 
96
                description='')
 
97
        )
 
98
        return d
 
99
 
 
100
    def test_power_change_failure_emits_event(self):
 
101
        system_id = factory.make_name('system_id')
 
102
        hostname = factory.make_name('hostname')
 
103
        message = factory.make_name('message')
 
104
        power_change = 'on'
 
105
        protocol, io = self.patch_rpc_methods()
 
106
        d = power.power_change_failure(
 
107
            system_id, hostname, power_change, message)
 
108
        io.flush()
 
109
        self.assertThat(
 
110
            protocol.SendEvent,
 
111
            MockCalledOnceWith(
 
112
                ANY,
 
113
                type_name=EVENT_TYPES.NODE_POWER_ON_FAILED,
 
114
                system_id=system_id,
 
115
                description=message)
 
116
        )
 
117
        return d
 
118
 
 
119
    def test_power_query_failure_emits_event(self):
 
120
        system_id = factory.make_name('system_id')
 
121
        hostname = factory.make_name('hostname')
 
122
        message = factory.make_name('message')
 
123
        protocol, io = self.patch_rpc_methods()
 
124
        d = power.power_query_failure(
 
125
            system_id, hostname, message)
 
126
        # This blocks until the deferred is complete
 
127
        io.flush()
 
128
        self.assertTrue(d.called)
 
129
        self.assertThat(
 
130
            protocol.SendEvent,
 
131
            MockCalledOnceWith(
 
132
                ANY,
 
133
                type_name=EVENT_TYPES.NODE_POWER_QUERY_FAILED,
 
134
                system_id=system_id,
 
135
                description=message)
 
136
        )
 
137
        return d
 
138
 
 
139
    def test_power_query_failure_marks_node_broken(self):
 
140
        system_id = factory.make_name('system_id')
 
141
        hostname = factory.make_name('hostname')
 
142
        message = factory.make_name('message')
 
143
        protocol, io = self.patch_rpc_methods()
 
144
        d = power.power_query_failure(
 
145
            system_id, hostname, message)
 
146
        # This blocks until the deferred is complete
 
147
        io.flush()
 
148
        self.assertTrue(d.called)
 
149
        self.assertThat(
 
150
            protocol.MarkNodeBroken,
 
151
            MockCalledOnceWith(
 
152
                ANY,
 
153
                system_id=system_id,
 
154
                error_description=message)
 
155
        )
 
156
        return d
 
157
 
 
158
    def test_power_state_update_calls_UpdateNodePowerState(self):
 
159
        system_id = factory.make_name('system_id')
 
160
        state = random.choice(['on', 'off'])
 
161
        protocol, io = self.patch_rpc_methods()
 
162
        d = power.power_state_update(
 
163
            system_id, state)
 
164
        # This blocks until the deferred is complete
 
165
        io.flush()
 
166
        self.assertTrue(d.called)
 
167
        self.assertThat(
 
168
            protocol.UpdateNodePowerState,
 
169
            MockCalledOnceWith(
 
170
                ANY,
 
171
                system_id=system_id,
 
172
                power_state=state)
 
173
        )
 
174
        return d
 
175
 
 
176
 
 
177
class TestChangePowerChange(MAASTestCase):
 
178
 
 
179
    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
 
180
 
 
181
    def setUp(self):
 
182
        super(TestChangePowerChange, self).setUp()
 
183
        self.patch(
 
184
            provisioningserver.rpc.power, 'deferToThread', maybeDeferred)
 
185
 
 
186
    def patch_power_action(self, return_value=None, side_effect=None):
 
187
        """Patch the PowerAction object.
 
188
 
 
189
        Patch the PowerAction object so that PowerAction().execute
 
190
        is replaced by a Mock object created using the given `return_value`
 
191
        and `side_effect`.
 
192
 
 
193
        This can be used to simulate various successes or failures patterns
 
194
        while manipulating the power state of a node.
 
195
 
 
196
        Returns a tuple of mock objects: power.PowerAction and
 
197
        power.PowerAction().execute.
 
198
        """
 
199
        power_action_obj = Mock()
 
200
        power_action_obj_execute = Mock(
 
201
            return_value=return_value, side_effect=side_effect)
 
202
        power_action_obj.execute = power_action_obj_execute
 
203
        power_action = self.patch(power, 'PowerAction')
 
204
        power_action.return_value = power_action_obj
 
205
        return power_action, power_action_obj_execute
 
206
 
 
207
    def patch_rpc_methods(self, return_value={}, side_effect=None):
 
208
        fixture = self.useFixture(MockClusterToRegionRPCFixture())
 
209
        protocol, io = fixture.makeEventLoop(
 
210
            region.MarkNodeBroken, region.UpdateNodePowerState,
 
211
            region.SendEvent)
 
212
        protocol.MarkNodeBroken.return_value = return_value
 
213
        protocol.MarkNodeBroken.side_effect = side_effect
 
214
        return protocol.MarkNodeBroken, io
 
215
 
 
216
    def test_change_power_state_changes_power_state(self):
 
217
        system_id = factory.make_name('system_id')
 
218
        hostname = factory.make_name('hostname')
 
219
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
220
        power_change = random.choice(['on', 'off'])
 
221
        context = {
 
222
            factory.make_name('context-key'): factory.make_name('context-val')
 
223
        }
 
224
        self.patch(power, 'pause')
 
225
        # Patch the power action utility so that it says the node is
 
226
        # in the required power state.
 
227
        power_action, execute = self.patch_power_action(
 
228
            return_value=power_change)
 
229
        markNodeBroken, io = self.patch_rpc_methods()
 
230
 
 
231
        d = power.change_power_state(
 
232
            system_id, hostname, power_type, power_change, context)
 
233
        io.flush()
 
234
        self.assertThat(
 
235
            execute,
 
236
            MockCallsMatch(
 
237
                # One call to change the power state.
 
238
                call(power_change=power_change, **context),
 
239
                # One call to query the power state.
 
240
                call(power_change='query', **context),
 
241
            ),
 
242
        )
 
243
        # The node hasn't been marked broken.
 
244
        self.assertThat(markNodeBroken, MockNotCalled())
 
245
        return d
 
246
 
 
247
    def test_change_power_state_doesnt_retry_for_certain_power_types(self):
 
248
        system_id = factory.make_name('system_id')
 
249
        hostname = factory.make_name('hostname')
 
250
        # Use a power type that is not among power.QUERY_POWER_TYPES.
 
251
        power_type = factory.make_name('power_type')
 
252
        power_change = random.choice(['on', 'off'])
 
253
        context = {
 
254
            factory.make_name('context-key'): factory.make_name('context-val')
 
255
        }
 
256
        self.patch(power, 'pause')
 
257
        power_action, execute = self.patch_power_action(
 
258
            return_value=random.choice(['on', 'off']))
 
259
        markNodeBroken, io = self.patch_rpc_methods()
 
260
 
 
261
        d = power.change_power_state(
 
262
            system_id, hostname, power_type, power_change, context)
 
263
        io.flush()
 
264
        self.assertThat(
 
265
            execute,
 
266
            MockCallsMatch(
 
267
                # Only one call to change the power state.
 
268
                call(power_change=power_change, **context),
 
269
            ),
 
270
        )
 
271
        # The node hasn't been marked broken.
 
272
        self.assertThat(markNodeBroken, MockNotCalled())
 
273
        return d
 
274
 
 
275
    def test_change_power_state_retries_if_power_state_doesnt_change(self):
 
276
        system_id = factory.make_name('system_id')
 
277
        hostname = factory.make_name('hostname')
 
278
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
279
        power_change = 'on'
 
280
        context = {
 
281
            factory.make_name('context-key'): factory.make_name('context-val')
 
282
        }
 
283
        self.patch(power, 'pause')
 
284
        # Simulate a failure to power up the node, then a success.
 
285
        power_action, execute = self.patch_power_action(
 
286
            side_effect=[None, 'off', None, 'on'])
 
287
        markNodeBroken, io = self.patch_rpc_methods()
 
288
 
 
289
        d = power.change_power_state(
 
290
            system_id, hostname, power_type, power_change, context)
 
291
        io.flush()
 
292
        self.assertThat(
 
293
            execute,
 
294
            MockCallsMatch(
 
295
                call(power_change=power_change, **context),
 
296
                call(power_change='query', **context),
 
297
                call(power_change=power_change, **context),
 
298
                call(power_change='query', **context),
 
299
            )
 
300
        )
 
301
        # The node hasn't been marked broken.
 
302
        self.assertThat(markNodeBroken, MockNotCalled())
 
303
        return d
 
304
 
 
305
    def test_change_power_state_marks_the_node_broken_if_failure(self):
 
306
        system_id = factory.make_name('system_id')
 
307
        hostname = factory.make_name('hostname')
 
308
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
309
        power_change = 'on'
 
310
        context = {
 
311
            factory.make_name('context-key'): factory.make_name('context-val')
 
312
        }
 
313
        self.patch(power, 'pause')
 
314
        # Simulate a persistent failure.
 
315
        power_action, execute = self.patch_power_action(return_value='off')
 
316
        markNodeBroken, io = self.patch_rpc_methods()
 
317
 
 
318
        d = power.change_power_state(
 
319
            system_id, hostname, power_type, power_change, context)
 
320
        io.flush()
 
321
 
 
322
        # The node has been marked broken.
 
323
        msg = "Timeout after %s tries" % len(
 
324
            power.default_waiting_policy)
 
325
        self.assertThat(
 
326
            markNodeBroken,
 
327
            MockCalledOnceWith(
 
328
                ANY,
 
329
                system_id=system_id,
 
330
                error_description=msg)
 
331
        )
 
332
        return d
 
333
 
 
334
    def test_change_power_state_marks_the_node_broken_if_exception(self):
 
335
        system_id = factory.make_name('system_id')
 
336
        hostname = factory.make_name('hostname')
 
337
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
338
        power_change = 'on'
 
339
        context = {
 
340
            factory.make_name('context-key'): factory.make_name('context-val')
 
341
        }
 
342
        self.patch(power, 'pause')
 
343
        # Simulate an exception.
 
344
        exception_message = factory.make_name('exception')
 
345
        power_action, execute = self.patch_power_action(
 
346
            side_effect=PowerActionFail(exception_message))
 
347
        markNodeBroken, io = self.patch_rpc_methods()
 
348
 
 
349
        d = power.change_power_state(
 
350
            system_id, hostname, power_type, power_change, context)
 
351
        io.flush()
 
352
        assert_fails_with(d, PowerActionFail)
 
353
        error_message = "Node could not be powered on: %s" % exception_message
 
354
 
 
355
        def check(failure):
 
356
            self.assertThat(
 
357
                markNodeBroken,
 
358
                MockCalledOnceWith(
 
359
                    ANY, system_id=system_id, error_description=error_message))
 
360
 
 
361
        return d.addCallback(check)
 
362
 
 
363
    def test_change_power_state_pauses_inbetween_retries(self):
 
364
        system_id = factory.make_name('system_id')
 
365
        hostname = factory.make_name('hostname')
 
366
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
367
        power_change = 'on'
 
368
        context = {
 
369
            factory.make_name('context-key'): factory.make_name('context-val')
 
370
        }
 
371
        # Simulate two failures to power up the node, then a success.
 
372
        power_action, execute = self.patch_power_action(
 
373
            side_effect=[None, 'off', None, 'off', None, 'on'])
 
374
        self.patch(power, "deferToThread", maybeDeferred)
 
375
        markNodeBroken, io = self.patch_rpc_methods()
 
376
        clock = Clock()
 
377
 
 
378
        calls_and_pause = [
 
379
            ([
 
380
                call(power_change=power_change, **context),
 
381
            ], 3),
 
382
            ([
 
383
                call(power_change='query', **context),
 
384
                call(power_change=power_change, **context),
 
385
            ], 5),
 
386
            ([
 
387
                call(power_change='query', **context),
 
388
                call(power_change=power_change, **context),
 
389
            ], 10),
 
390
            ([
 
391
                call(power_change='query', **context),
 
392
            ], 0),
 
393
        ]
 
394
        calls = []
 
395
        d = power.change_power_state(
 
396
            system_id, hostname, power_type, power_change, context,
 
397
            clock=clock)
 
398
        for newcalls, waiting_time in calls_and_pause:
 
399
            calls.extend(newcalls)
 
400
            io.flush()
 
401
            self.assertThat(execute, MockCallsMatch(*calls))
 
402
            clock.advance(waiting_time)
 
403
        return d
 
404
 
 
405
 
 
406
class TestPowerQuery(MAASTestCase):
 
407
 
 
408
    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
 
409
 
 
410
    def setUp(self):
 
411
        super(TestPowerQuery, self).setUp()
 
412
        self.patch(
 
413
            provisioningserver.rpc.power, 'deferToThread', maybeDeferred)
 
414
 
 
415
    def patch_power_action(self, return_value=None, side_effect=None):
 
416
        """Patch the PowerAction object.
 
417
 
 
418
        Patch the PowerAction object so that PowerAction().execute
 
419
        is replaced by a Mock object created using the given `return_value`
 
420
        and `side_effect`.
 
421
 
 
422
        This can be used to simulate various successes or failures patterns
 
423
        while performing operations on the node.
 
424
 
 
425
        Returns a tuple of mock objects: power.PowerAction and
 
426
        power.PowerAction().execute.
 
427
        """
 
428
        power_action_obj = Mock()
 
429
        power_action_obj_execute = Mock(
 
430
            return_value=return_value, side_effect=side_effect)
 
431
        power_action_obj.execute = power_action_obj_execute
 
432
        power_action = self.patch(power, 'PowerAction')
 
433
        power_action.return_value = power_action_obj
 
434
        return power_action, power_action_obj_execute
 
435
 
 
436
    def patch_rpc_methods(self, return_value={}, side_effect=None):
 
437
        fixture = self.useFixture(MockClusterToRegionRPCFixture())
 
438
        protocol, io = fixture.makeEventLoop(
 
439
            region.MarkNodeBroken, region.SendEvent)
 
440
        protocol.MarkNodeBroken.return_value = return_value
 
441
        protocol.MarkNodeBroken.side_effect = side_effect
 
442
        return protocol.SendEvent, protocol.MarkNodeBroken, io
 
443
 
 
444
    def test_get_power_state_querys_node(self):
 
445
        system_id = factory.make_name('system_id')
 
446
        hostname = factory.make_name('hostname')
 
447
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
448
        power_state = random.choice(['on', 'off'])
 
449
        context = {
 
450
            factory.make_name('context-key'): factory.make_name('context-val')
 
451
        }
 
452
        self.patch(power, 'pause')
 
453
        # Patch the power action utility so that it says the node is
 
454
        # in on/off power state.
 
455
        power_action, execute = self.patch_power_action(
 
456
            return_value=power_state)
 
457
        _, markNodeBroken, io = self.patch_rpc_methods()
 
458
 
 
459
        d = power.get_power_state(
 
460
            system_id, hostname, power_type, context)
 
461
        # This blocks until the deferred is complete
 
462
        io.flush()
 
463
        self.assertTrue(d.called)
 
464
        self.assertThat(
 
465
            execute,
 
466
            MockCallsMatch(
 
467
                # One call to change the power state.
 
468
                call(power_change='query', **context),
 
469
            ),
 
470
        )
 
471
        self.assertEqual(power_state, d.result)
 
472
        return d
 
473
 
 
474
    def test_get_power_state_returns_unknown_for_certain_power_types(self):
 
475
        system_id = factory.make_name('system_id')
 
476
        hostname = factory.make_name('hostname')
 
477
        # Use a power type that is not among power.QUERY_POWER_TYPES.
 
478
        power_type = factory.make_name('power_type')
 
479
        context = {
 
480
            factory.make_name('context-key'): factory.make_name('context-val')
 
481
        }
 
482
        _, _, io = self.patch_rpc_methods()
 
483
 
 
484
        d = power.get_power_state(
 
485
            system_id, hostname, power_type, context)
 
486
        # This blocks until the deferred is complete
 
487
        io.flush()
 
488
        self.assertTrue(d.called)
 
489
        self.assertEqual('unknown', d.result)
 
490
        return d
 
491
 
 
492
    def test_get_power_state_retries_if_power_query_fails(self):
 
493
        system_id = factory.make_name('system_id')
 
494
        hostname = factory.make_name('hostname')
 
495
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
496
        power_state = random.choice(['on', 'off'])
 
497
        err_msg = factory.make_name('error')
 
498
        context = {
 
499
            factory.make_name('context-key'): factory.make_name('context-val')
 
500
        }
 
501
        self.patch(power, 'pause')
 
502
        # Simulate a failure to power query the node, then success.
 
503
        power_action, execute = self.patch_power_action(
 
504
            side_effect=[PowerActionFail(err_msg), power_state])
 
505
        sendEvent, markNodeBroken, io = self.patch_rpc_methods()
 
506
 
 
507
        d = power.get_power_state(
 
508
            system_id, hostname, power_type, context)
 
509
        # This blocks until the deferred is complete
 
510
        io.flush()
 
511
        self.assertTrue(d.called)
 
512
        self.assertThat(
 
513
            execute,
 
514
            MockCallsMatch(
 
515
                call(power_change='query', **context),
 
516
                call(power_change='query', **context),
 
517
            )
 
518
        )
 
519
        # The node hasn't been marked broken.
 
520
        self.assertThat(markNodeBroken, MockNotCalled())
 
521
        self.assertEqual(power_state, d.result)
 
522
        return d
 
523
 
 
524
    def test_get_power_state_marks_the_node_broken_if_failure(self):
 
525
        system_id = factory.make_name('system_id')
 
526
        hostname = factory.make_name('hostname')
 
527
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
528
        err_msg = factory.make_name('error')
 
529
        context = {
 
530
            factory.make_name('context-key'): factory.make_name('context-val')
 
531
        }
 
532
        self.patch(power, 'pause')
 
533
        # Simulate a persistent failure.
 
534
        power_action, execute = self.patch_power_action(
 
535
            side_effect=PowerActionFail(err_msg))
 
536
        _, markNodeBroken, io = self.patch_rpc_methods()
 
537
 
 
538
        d = power.get_power_state(
 
539
            system_id, hostname, power_type, context)
 
540
        # This blocks until the deferred is complete
 
541
        io.flush()
 
542
        self.assertTrue(d.called)
 
543
        # The node has been marked broken.
 
544
        self.assertThat(
 
545
            markNodeBroken,
 
546
            MockCalledOnceWith(
 
547
                ANY,
 
548
                system_id=system_id,
 
549
                error_description="Node could not be queried %s (%s) %s" % (
 
550
                    system_id, hostname, err_msg))
 
551
        )
 
552
        self.assertEqual('error', d.result)
 
553
        return d
 
554
 
 
555
    def test_get_power_state_pauses_inbetween_retries(self):
 
556
        system_id = factory.make_name('system_id')
 
557
        hostname = factory.make_name('hostname')
 
558
        power_type = random.choice(power.QUERY_POWER_TYPES)
 
559
        context = {
 
560
            factory.make_name('context-key'): factory.make_name('context-val')
 
561
        }
 
562
        # Simulate two failures to power up the node, then a success.
 
563
        power_action, execute = self.patch_power_action(
 
564
            side_effect=[PowerActionFail, PowerActionFail, 'off'])
 
565
        self.patch(power, "deferToThread", maybeDeferred)
 
566
        _, _, io = self.patch_rpc_methods()
 
567
        clock = Clock()
 
568
 
 
569
        calls_and_pause = [
 
570
            ([
 
571
                call(power_change='query', **context),
 
572
            ], 3),
 
573
            ([
 
574
                call(power_change='query', **context),
 
575
            ], 5),
 
576
            ([
 
577
                call(power_change='query', **context),
 
578
            ], 10),
 
579
        ]
 
580
        calls = []
 
581
        d = power.get_power_state(
 
582
            system_id, hostname, power_type, context, clock=clock)
 
583
        for newcalls, waiting_time in calls_and_pause:
 
584
            calls.extend(newcalls)
 
585
            # This blocks until the deferred is complete
 
586
            io.flush()
 
587
            self.assertThat(execute, MockCallsMatch(*calls))
 
588
            clock.advance(waiting_time)
 
589
        return d
 
590
 
 
591
    def make_nodes(self):
 
592
        nodes = []
 
593
        for node in nodes:
 
594
            system_id = factory.make_name('system_id')
 
595
            hostname = factory.make_name('hostname')
 
596
            power_type = random.choice(power.QUERY_POWER_TYPES)
 
597
            state = random.choice(['on', 'off', 'unknown', 'error'])
 
598
            context = {
 
599
                factory.make_name(
 
600
                    'context-key'): factory.make_name('context-val')
 
601
            }
 
602
            nodes.append({
 
603
                'system_id': system_id,
 
604
                'hostname': hostname,
 
605
                'power_type': power_type,
 
606
                'context': context,
 
607
                'state': state,
 
608
                })
 
609
        return nodes
 
610
 
 
611
    def pick_alternate_state(self, state):
 
612
        return random.choice([
 
613
            value for value in ['on', 'off', 'unknown', 'error']
 
614
            if value != state])
 
615
 
 
616
    def test_query_all_nodes_calls_get_power_state(self):
 
617
        nodes = self.make_nodes()
 
618
        states = [node['state'] for node in nodes]
 
619
        get_state = self.patch(power, 'get_power_state')
 
620
        get_state.side_effect = states
 
621
 
 
622
        calls = []
 
623
        for node in nodes:
 
624
            calls.append(
 
625
                call(
 
626
                    node['system_id'], node['hostname'],
 
627
                    node['power_type'], node['context']))
 
628
 
 
629
        self.assertThat(get_state, MockCallsMatch(*calls))
 
630
 
 
631
    def test_query_all_nodes_calls_power_state_update(self):
 
632
        nodes = self.make_nodes()
 
633
        states = [self.pick_alternate_state(node['state']) for node in nodes]
 
634
        get_state = self.patch(power, 'get_power_state')
 
635
        get_state.side_effect = states
 
636
        update_state = self.patch(power, 'power_state_update')
 
637
 
 
638
        calls = []
 
639
        for i in range(len(nodes)):
 
640
            node = nodes[i]
 
641
            new_state = states[i]
 
642
            calls.append(
 
643
                call(
 
644
                    node['system_id'], new_state))
 
645
 
 
646
        self.assertThat(update_state, MockCallsMatch(*calls))