← Back to branch summary

~nskaggs/+junk/xenial-test

« back to all changes in this revision

Viewing changes to src/github.com/juju/juju/worker/upgradesteps/worker_test.go

  • Committer: Nicholas Skaggs
  • Date: 2016-10-24 20:56:05 UTC
  • Revision ID: nicholas.skaggs@canonical.com-20161024205605-z8lta0uvuhtxwzwl
Initi with beta15

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/github.com/juju/juju/worker/upgradesteps/worker_test.go
 
1
// Copyright 2015 Canonical Ltd.
 
2
// Licensed under the AGPLv3, see LICENCE file for details.
 
3
 
 
4
package upgradesteps
 
5
 
 
6
import (
 
7
        "fmt"
 
8
        "time"
 
9
 
 
10
        "github.com/juju/errors"
 
11
        "github.com/juju/loggo"
 
12
        jc "github.com/juju/testing/checkers"
 
13
        "github.com/juju/utils"
 
14
        "github.com/juju/utils/arch"
 
15
        "github.com/juju/utils/series"
 
16
        gc "gopkg.in/check.v1"
 
17
        "gopkg.in/juju/names.v2"
 
18
 
 
19
        "github.com/juju/juju/agent"
 
20
        cmdutil "github.com/juju/juju/cmd/jujud/util"
 
21
        "github.com/juju/juju/constraints"
 
22
        "github.com/juju/juju/environs"
 
23
        "github.com/juju/juju/instance"
 
24
        "github.com/juju/juju/mongo/mongotest"
 
25
        "github.com/juju/juju/state"
 
26
        "github.com/juju/juju/state/multiwatcher"
 
27
        "github.com/juju/juju/state/stateenvirons"
 
28
        statetesting "github.com/juju/juju/state/testing"
 
29
        "github.com/juju/juju/status"
 
30
        coretesting "github.com/juju/juju/testing"
 
31
        "github.com/juju/juju/testing/factory"
 
32
        "github.com/juju/juju/upgrades"
 
33
        jujuversion "github.com/juju/juju/version"
 
34
        "github.com/juju/juju/worker"
 
35
        "github.com/juju/juju/worker/gate"
 
36
        "github.com/juju/version"
 
37
)
 
38
 
 
39
// TODO(mjs) - these tests are too tightly coupled to the
 
40
// implementation. They needn't be internal tests.
 
41
 
 
42
type UpgradeSuite struct {
 
43
        statetesting.StateSuite
 
44
 
 
45
        oldVersion      version.Binary
 
46
        logWriter       loggo.TestWriter
 
47
        connectionDead  bool
 
48
        machineIsMaster bool
 
49
        preUpgradeError bool
 
50
}
 
51
 
 
52
var _ = gc.Suite(&UpgradeSuite{})
 
53
 
 
54
const fails = true
 
55
const succeeds = false
 
56
 
 
57
func (s *UpgradeSuite) SetUpTest(c *gc.C) {
 
58
        s.StateSuite.SetUpTest(c)
 
59
 
 
60
        s.preUpgradeError = false
 
61
        // Most of these tests normally finish sub-second on a fast machine.
 
62
        // If any given test hits a minute, we have almost certainly become
 
63
        // wedged, so dump the logs.
 
64
        coretesting.DumpTestLogsAfter(time.Minute, c, s)
 
65
 
 
66
        s.oldVersion = version.Binary{
 
67
                Number: jujuversion.Current,
 
68
                Arch:   arch.HostArch(),
 
69
                Series: series.HostSeries(),
 
70
        }
 
71
        s.oldVersion.Major = 1
 
72
        s.oldVersion.Minor = 16
 
73
 
 
74
        // Don't wait so long in tests.
 
75
        s.PatchValue(&UpgradeStartTimeoutMaster, time.Duration(time.Millisecond*50))
 
76
        s.PatchValue(&UpgradeStartTimeoutSecondary, time.Duration(time.Millisecond*60))
 
77
 
 
78
        // Allow tests to make the API connection appear to be dead.
 
79
        s.connectionDead = false
 
80
        s.PatchValue(&cmdutil.ConnectionIsDead, func(loggo.Logger, cmdutil.Pinger) bool {
 
81
                return s.connectionDead
 
82
        })
 
83
 
 
84
        s.machineIsMaster = true
 
85
        fakeIsMachineMaster := func(*state.State, string) (bool, error) {
 
86
                return s.machineIsMaster, nil
 
87
        }
 
88
        s.PatchValue(&IsMachineMaster, fakeIsMachineMaster)
 
89
 
 
90
}
 
91
 
 
92
func (s *UpgradeSuite) captureLogs(c *gc.C) {
 
93
        c.Assert(loggo.RegisterWriter("upgrade-tests", &s.logWriter), gc.IsNil)
 
94
        s.AddCleanup(func(*gc.C) {
 
95
                loggo.RemoveWriter("upgrade-tests")
 
96
                s.logWriter.Clear()
 
97
        })
 
98
}
 
99
 
 
100
func (s *UpgradeSuite) countUpgradeAttempts(upgradeErr error) *int {
 
101
        count := 0
 
102
        s.PatchValue(&PerformUpgrade, func(version.Number, []upgrades.Target, upgrades.Context) error {
 
103
                count++
 
104
                return upgradeErr
 
105
        })
 
106
        return &count
 
107
}
 
108
 
 
109
func (s *UpgradeSuite) TestNewChannelWhenNoUpgradeRequired(c *gc.C) {
 
110
        // Set the agent's initial upgradedToVersion to almost the same as
 
111
        // the current version. We want it to be different to
 
112
        // jujuversion.Current (so that we can see it change) but not to
 
113
        // trigger upgrade steps.
 
114
        config := NewFakeConfigSetter(names.NewMachineTag("0"), makeBumpedCurrentVersion().Number)
 
115
        agent := NewFakeAgent(config)
 
116
 
 
117
        lock, err := NewLock(agent)
 
118
        c.Assert(err, jc.ErrorIsNil)
 
119
 
 
120
        c.Assert(lock.IsUnlocked(), jc.IsTrue)
 
121
        // The agent's version should have been updated.
 
122
        c.Assert(config.Version, gc.Equals, jujuversion.Current)
 
123
 
 
124
}
 
125
 
 
126
func (s *UpgradeSuite) TestNewChannelWhenUpgradeRequired(c *gc.C) {
 
127
        // Set the agent's upgradedToVersion so that upgrade steps are required.
 
128
        initialVersion := version.MustParse("1.16.0")
 
129
        config := NewFakeConfigSetter(names.NewMachineTag("0"), initialVersion)
 
130
        agent := NewFakeAgent(config)
 
131
 
 
132
        lock, err := NewLock(agent)
 
133
        c.Assert(err, jc.ErrorIsNil)
 
134
 
 
135
        c.Assert(lock.IsUnlocked(), jc.IsFalse)
 
136
        // The agent's version should NOT have been updated.
 
137
        c.Assert(config.Version, gc.Equals, initialVersion)
 
138
}
 
139
 
 
140
func (s *UpgradeSuite) TestRetryStrategy(c *gc.C) {
 
141
        retries := getUpgradeRetryStrategy()
 
142
        c.Assert(retries.Delay, gc.Equals, 2*time.Minute)
 
143
        c.Assert(retries.Min, gc.Equals, 5)
 
144
}
 
145
 
 
146
func (s *UpgradeSuite) TestNoUpgradeNecessary(c *gc.C) {
 
147
        attemptsP := s.countUpgradeAttempts(nil)
 
148
        s.captureLogs(c)
 
149
        s.oldVersion.Number = jujuversion.Current // nothing to do
 
150
 
 
151
        workerErr, config, _, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
 
152
 
 
153
        c.Check(workerErr, gc.IsNil)
 
154
        c.Check(*attemptsP, gc.Equals, 0)
 
155
        c.Check(config.Version, gc.Equals, jujuversion.Current)
 
156
        c.Check(doneLock.IsUnlocked(), jc.IsTrue)
 
157
}
 
158
 
 
159
func (s *UpgradeSuite) TestUpgradeStepsFailure(c *gc.C) {
 
160
        // This test checks what happens when every upgrade attempt fails.
 
161
        // A number of retries should be observed and the agent should end
 
162
        // up in a state where it is is still running but is reporting an
 
163
        // error and the upgrade is not flagged as having completed (which
 
164
        // prevents most of the agent's workers from running and keeps the
 
165
        // API in restricted mode).
 
166
 
 
167
        attemptsP := s.countUpgradeAttempts(errors.New("boom"))
 
168
        s.captureLogs(c)
 
169
 
 
170
        workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
 
171
 
 
172
        // The worker shouldn't return an error so that the worker and
 
173
        // agent keep running.
 
174
        c.Check(workerErr, gc.IsNil)
 
175
 
 
176
        c.Check(*attemptsP, gc.Equals, maxUpgradeRetries)
 
177
        c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish
 
178
        c.Assert(statusCalls, jc.DeepEquals,
 
179
                s.makeExpectedStatusCalls(maxUpgradeRetries-1, fails, "boom"))
 
180
        c.Assert(s.logWriter.Log(), jc.LogMatches,
 
181
                s.makeExpectedUpgradeLogs(maxUpgradeRetries-1, "hostMachine", fails, "boom"))
 
182
        c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
 
183
}
 
184
 
 
185
func (s *UpgradeSuite) TestUpgradeStepsRetries(c *gc.C) {
 
186
        // This test checks what happens when the first upgrade attempt
 
187
        // fails but the following on succeeds. The final state should be
 
188
        // the same as a successful upgrade which worked first go.
 
189
        attempts := 0
 
190
        fail := true
 
191
        fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error {
 
192
                attempts++
 
193
                if fail {
 
194
                        fail = false
 
195
                        return errors.New("boom")
 
196
                } else {
 
197
                        return nil
 
198
                }
 
199
        }
 
200
        s.PatchValue(&PerformUpgrade, fakePerformUpgrade)
 
201
        s.captureLogs(c)
 
202
 
 
203
        workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
 
204
 
 
205
        c.Check(workerErr, gc.IsNil)
 
206
        c.Check(attempts, gc.Equals, 2)
 
207
        c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade finished
 
208
        c.Assert(statusCalls, jc.DeepEquals, s.makeExpectedStatusCalls(1, succeeds, "boom"))
 
209
        c.Assert(s.logWriter.Log(), jc.LogMatches, s.makeExpectedUpgradeLogs(1, "hostMachine", succeeds, "boom"))
 
210
        c.Check(doneLock.IsUnlocked(), jc.IsTrue)
 
211
}
 
212
 
 
213
func (s *UpgradeSuite) TestOtherUpgradeRunFailure(c *gc.C) {
 
214
        // This test checks what happens something other than the upgrade
 
215
        // steps themselves fails, ensuring the something is logged and
 
216
        // the agent status is updated.
 
217
 
 
218
        fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error {
 
219
                // Delete UpgradeInfo for the upgrade so that finaliseUpgrade() will fail
 
220
                s.State.ClearUpgradeInfo()
 
221
                return nil
 
222
        }
 
223
        s.PatchValue(&PerformUpgrade, fakePerformUpgrade)
 
224
        s.Factory.MakeMachine(c, &factory.MachineParams{
 
225
                Jobs: []state.MachineJob{state.JobManageModel},
 
226
        })
 
227
        s.captureLogs(c)
 
228
 
 
229
        workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel)
 
230
 
 
231
        c.Check(workerErr, gc.IsNil)
 
232
        c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade almost finished
 
233
        failReason := `upgrade done but: cannot set upgrade status to "finishing": ` +
 
234
                `Another status change may have occurred concurrently`
 
235
        c.Assert(statusCalls, jc.DeepEquals,
 
236
                s.makeExpectedStatusCalls(0, fails, failReason))
 
237
        c.Assert(s.logWriter.Log(), jc.LogMatches,
 
238
                s.makeExpectedUpgradeLogs(0, "databaseMaster", fails, failReason))
 
239
        c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
 
240
}
 
241
 
 
242
func (s *UpgradeSuite) TestApiConnectionFailure(c *gc.C) {
 
243
        // This test checks what happens when an upgrade fails because the
 
244
        // connection to mongo has gone away. This will happen when the
 
245
        // mongo master changes. In this case we want the upgrade worker
 
246
        // to return immediately without further retries. The error should
 
247
        // be returned by the worker so that the agent will restart.
 
248
 
 
249
        attemptsP := s.countUpgradeAttempts(errors.New("boom"))
 
250
        s.connectionDead = true // Make the connection to state appear to be dead
 
251
        s.captureLogs(c)
 
252
 
 
253
        workerErr, config, _, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
 
254
 
 
255
        c.Check(workerErr, gc.ErrorMatches, "API connection lost during upgrade: boom")
 
256
        c.Check(*attemptsP, gc.Equals, 1)
 
257
        c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish
 
258
        c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
 
259
}
 
260
 
 
261
func (s *UpgradeSuite) TestAbortWhenOtherControllerDoesntStartUpgrade(c *gc.C) {
 
262
        // This test checks when a controller is upgrading and one of
 
263
        // the other controllers doesn't signal it is ready in time.
 
264
 
 
265
        err := s.State.SetModelAgentVersion(jujuversion.Current)
 
266
        c.Assert(err, jc.ErrorIsNil)
 
267
 
 
268
        // The master controller in this scenario is functionally tested
 
269
        // elsewhere.
 
270
        s.machineIsMaster = false
 
271
 
 
272
        s.create3Controllers(c)
 
273
        s.captureLogs(c)
 
274
        attemptsP := s.countUpgradeAttempts(nil)
 
275
 
 
276
        workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel)
 
277
 
 
278
        c.Check(workerErr, gc.IsNil)
 
279
        c.Check(*attemptsP, gc.Equals, 0)
 
280
        c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't happen
 
281
        c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
 
282
 
 
283
        // The environment agent-version should still be the new version.
 
284
        // It's up to the master to trigger the rollback.
 
285
        s.assertEnvironAgentVersion(c, jujuversion.Current)
 
286
 
 
287
        causeMsg := " timed out after 60ms"
 
288
        c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{
 
289
                {loggo.INFO, "waiting for other controllers to be ready for upgrade"},
 
290
                {loggo.ERROR, "aborted wait for other controllers: timed out after 60ms"},
 
291
                {loggo.ERROR, `upgrade from .+ to .+ for "machine-0" failed \(giving up\): ` +
 
292
                        "aborted wait for other controllers:" + causeMsg},
 
293
        })
 
294
        c.Assert(statusCalls, jc.DeepEquals, []StatusCall{{
 
295
                status.StatusError,
 
296
                fmt.Sprintf(
 
297
                        "upgrade to %s failed (giving up): aborted wait for other controllers:"+causeMsg,
 
298
                        jujuversion.Current),
 
299
        }})
 
300
}
 
301
 
 
302
func (s *UpgradeSuite) TestSuccessMaster(c *gc.C) {
 
303
        // This test checks what happens when an upgrade works on the
 
304
        // first attempt on a master controller.
 
305
        s.machineIsMaster = true
 
306
        info := s.checkSuccess(c, "databaseMaster", func(*state.UpgradeInfo) {})
 
307
        c.Assert(info.Status(), gc.Equals, state.UpgradeFinishing)
 
308
}
 
309
 
 
310
func (s *UpgradeSuite) TestSuccessSecondary(c *gc.C) {
 
311
        // This test checks what happens when an upgrade works on the
 
312
        // first attempt on a secondary controller.
 
313
        s.machineIsMaster = false
 
314
        mungeInfo := func(info *state.UpgradeInfo) {
 
315
                // Indicate that the master is done
 
316
                err := info.SetStatus(state.UpgradeRunning)
 
317
                c.Assert(err, jc.ErrorIsNil)
 
318
                err = info.SetStatus(state.UpgradeFinishing)
 
319
                c.Assert(err, jc.ErrorIsNil)
 
320
        }
 
321
        s.checkSuccess(c, "controller", mungeInfo)
 
322
}
 
323
 
 
324
func (s *UpgradeSuite) checkSuccess(c *gc.C, target string, mungeInfo func(*state.UpgradeInfo)) *state.UpgradeInfo {
 
325
        _, machineIdB, machineIdC := s.create3Controllers(c)
 
326
 
 
327
        // Indicate that machine B and C are ready to upgrade
 
328
        vPrevious := s.oldVersion.Number
 
329
        vNext := jujuversion.Current
 
330
        info, err := s.State.EnsureUpgradeInfo(machineIdB, vPrevious, vNext)
 
331
        c.Assert(err, jc.ErrorIsNil)
 
332
        _, err = s.State.EnsureUpgradeInfo(machineIdC, vPrevious, vNext)
 
333
        c.Assert(err, jc.ErrorIsNil)
 
334
 
 
335
        mungeInfo(info)
 
336
 
 
337
        attemptsP := s.countUpgradeAttempts(nil)
 
338
        s.captureLogs(c)
 
339
 
 
340
        workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel)
 
341
 
 
342
        c.Check(workerErr, gc.IsNil)
 
343
        c.Check(*attemptsP, gc.Equals, 1)
 
344
        c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade finished
 
345
        c.Assert(statusCalls, jc.DeepEquals, s.makeExpectedStatusCalls(0, succeeds, ""))
 
346
        c.Assert(s.logWriter.Log(), jc.LogMatches, s.makeExpectedUpgradeLogs(0, target, succeeds, ""))
 
347
        c.Check(doneLock.IsUnlocked(), jc.IsTrue)
 
348
 
 
349
        err = info.Refresh()
 
350
        c.Assert(err, jc.ErrorIsNil)
 
351
        c.Assert(info.ControllersDone(), jc.DeepEquals, []string{"0"})
 
352
        return info
 
353
}
 
354
 
 
355
func (s *UpgradeSuite) TestJobsToTargets(c *gc.C) {
 
356
        check := func(jobs []multiwatcher.MachineJob, isMaster bool, expectedTargets ...upgrades.Target) {
 
357
                c.Assert(jobsToTargets(jobs, isMaster), jc.SameContents, expectedTargets)
 
358
        }
 
359
 
 
360
        check([]multiwatcher.MachineJob{multiwatcher.JobHostUnits}, false, upgrades.HostMachine)
 
361
        check([]multiwatcher.MachineJob{multiwatcher.JobManageModel}, false, upgrades.Controller)
 
362
        check([]multiwatcher.MachineJob{multiwatcher.JobManageModel}, true,
 
363
                upgrades.Controller, upgrades.DatabaseMaster)
 
364
        check([]multiwatcher.MachineJob{multiwatcher.JobManageModel, multiwatcher.JobHostUnits}, false,
 
365
                upgrades.Controller, upgrades.HostMachine)
 
366
        check([]multiwatcher.MachineJob{multiwatcher.JobManageModel, multiwatcher.JobHostUnits}, true,
 
367
                upgrades.Controller, upgrades.DatabaseMaster, upgrades.HostMachine)
 
368
}
 
369
 
 
370
func (s *UpgradeSuite) TestPreUpgradeFail(c *gc.C) {
 
371
        s.preUpgradeError = true
 
372
        s.captureLogs(c)
 
373
 
 
374
        workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
 
375
 
 
376
        c.Check(workerErr, jc.ErrorIsNil)
 
377
        c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish
 
378
        c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
 
379
 
 
380
        causeMessage := `machine 0 cannot be upgraded: preupgrade error`
 
381
        failMessage := fmt.Sprintf(
 
382
                `upgrade from %s to %s for "machine-0" failed \(giving up\): %s`,
 
383
                s.oldVersion.Number, jujuversion.Current, causeMessage)
 
384
        c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{
 
385
                {loggo.INFO, "checking that upgrade can proceed"},
 
386
                {loggo.ERROR, failMessage},
 
387
        })
 
388
 
 
389
        statusMessage := fmt.Sprintf(
 
390
                `upgrade to %s failed (giving up): %s`, jujuversion.Current, causeMessage)
 
391
        c.Assert(statusCalls, jc.DeepEquals, []StatusCall{{
 
392
                status.StatusError, statusMessage,
 
393
        }})
 
394
}
 
395
 
 
396
// Run just the upgradesteps worker with a fake machine agent and
 
397
// fake agent config.
 
398
func (s *UpgradeSuite) runUpgradeWorker(c *gc.C, jobs ...multiwatcher.MachineJob) (
 
399
        error, *fakeConfigSetter, []StatusCall, gate.Lock,
 
400
) {
 
401
        s.setInstantRetryStrategy(c)
 
402
        config := s.makeFakeConfig()
 
403
        agent := NewFakeAgent(config)
 
404
        doneLock, err := NewLock(agent)
 
405
        c.Assert(err, jc.ErrorIsNil)
 
406
        machineStatus := &testStatusSetter{}
 
407
        worker, err := NewWorker(doneLock, agent, nil, jobs, s.openStateForUpgrade, s.preUpgradeSteps, machineStatus)
 
408
        c.Assert(err, jc.ErrorIsNil)
 
409
        return worker.Wait(), config, machineStatus.Calls, doneLock
 
410
}
 
411
 
 
412
func (s *UpgradeSuite) openStateForUpgrade() (*state.State, error) {
 
413
        mongoInfo := s.State.MongoConnectionInfo()
 
414
        newPolicy := stateenvirons.GetNewPolicyFunc(
 
415
                stateenvirons.GetNewEnvironFunc(environs.New),
 
416
        )
 
417
        st, err := state.Open(s.State.ModelTag(), mongoInfo, mongotest.DialOpts(), newPolicy)
 
418
        if err != nil {
 
419
                return nil, err
 
420
        }
 
421
        return st, nil
 
422
}
 
423
 
 
424
func (s *UpgradeSuite) preUpgradeSteps(st *state.State, agentConf agent.Config, isController, isMasterController bool) error {
 
425
        if s.preUpgradeError {
 
426
                return errors.New("preupgrade error")
 
427
        }
 
428
        return nil
 
429
}
 
430
 
 
431
func (s *UpgradeSuite) makeFakeConfig() *fakeConfigSetter {
 
432
        return NewFakeConfigSetter(names.NewMachineTag("0"), s.oldVersion.Number)
 
433
}
 
434
 
 
435
func (s *UpgradeSuite) create3Controllers(c *gc.C) (machineIdA, machineIdB, machineIdC string) {
 
436
        machine0 := s.Factory.MakeMachine(c, &factory.MachineParams{
 
437
                Jobs: []state.MachineJob{state.JobManageModel},
 
438
        })
 
439
        machineIdA = machine0.Id()
 
440
        s.setMachineAlive(c, machineIdA)
 
441
 
 
442
        changes, err := s.State.EnableHA(3, constraints.Value{}, "quantal", nil)
 
443
        c.Assert(err, jc.ErrorIsNil)
 
444
        c.Assert(len(changes.Added), gc.Equals, 2)
 
445
 
 
446
        machineIdB = changes.Added[0]
 
447
        s.setMachineProvisioned(c, machineIdB)
 
448
        s.setMachineAlive(c, machineIdB)
 
449
 
 
450
        machineIdC = changes.Added[1]
 
451
        s.setMachineProvisioned(c, machineIdC)
 
452
        s.setMachineAlive(c, machineIdC)
 
453
 
 
454
        return
 
455
}
 
456
 
 
457
func (s *UpgradeSuite) setMachineProvisioned(c *gc.C, id string) {
 
458
        machine, err := s.State.Machine(id)
 
459
        c.Assert(err, jc.ErrorIsNil)
 
460
        err = machine.SetProvisioned(instance.Id(id+"-inst"), "nonce", nil)
 
461
        c.Assert(err, jc.ErrorIsNil)
 
462
}
 
463
 
 
464
func (s *UpgradeSuite) setMachineAlive(c *gc.C, id string) {
 
465
        machine, err := s.State.Machine(id)
 
466
        c.Assert(err, jc.ErrorIsNil)
 
467
        pinger, err := machine.SetAgentPresence()
 
468
        c.Assert(err, jc.ErrorIsNil)
 
469
        s.AddCleanup(func(c *gc.C) {
 
470
                c.Assert(worker.Stop(pinger), jc.ErrorIsNil)
 
471
        })
 
472
}
 
473
 
 
474
// Return a version the same as the current software version, but with
 
475
// the build number bumped.
 
476
//
 
477
// The version Tag is also cleared so that upgrades.PerformUpgrade
 
478
// doesn't think it needs to run upgrade steps unnecessarily.
 
479
func makeBumpedCurrentVersion() version.Binary {
 
480
        v := version.Binary{
 
481
                Number: jujuversion.Current,
 
482
                Arch:   arch.HostArch(),
 
483
                Series: series.HostSeries(),
 
484
        }
 
485
        v.Build++
 
486
        v.Tag = ""
 
487
        return v
 
488
}
 
489
 
 
490
const maxUpgradeRetries = 3
 
491
 
 
492
func (s *UpgradeSuite) setInstantRetryStrategy(c *gc.C) {
 
493
        // TODO(katco): 2016-08-09: lp:1611427
 
494
        s.PatchValue(&getUpgradeRetryStrategy, func() utils.AttemptStrategy {
 
495
                c.Logf("setting instant retry strategy for upgrade: retries=%d", maxUpgradeRetries)
 
496
                return utils.AttemptStrategy{
 
497
                        Delay: 0,
 
498
                        Min:   maxUpgradeRetries,
 
499
                }
 
500
        })
 
501
}
 
502
 
 
503
func (s *UpgradeSuite) makeExpectedStatusCalls(retryCount int, expectFail bool, failReason string) []StatusCall {
 
504
        calls := []StatusCall{{
 
505
                status.StatusStarted,
 
506
                fmt.Sprintf("upgrading to %s", jujuversion.Current),
 
507
        }}
 
508
        for i := 0; i < retryCount; i++ {
 
509
                calls = append(calls, StatusCall{
 
510
                        status.StatusError,
 
511
                        fmt.Sprintf("upgrade to %s failed (will retry): %s", jujuversion.Current, failReason),
 
512
                })
 
513
        }
 
514
        if expectFail {
 
515
                calls = append(calls, StatusCall{
 
516
                        status.StatusError,
 
517
                        fmt.Sprintf("upgrade to %s failed (giving up): %s", jujuversion.Current, failReason),
 
518
                })
 
519
        } else {
 
520
                calls = append(calls, StatusCall{status.StatusStarted, ""})
 
521
        }
 
522
        return calls
 
523
}
 
524
 
 
525
func (s *UpgradeSuite) makeExpectedUpgradeLogs(retryCount int, target string, expectFail bool, failReason string) []jc.SimpleMessage {
 
526
        outLogs := []jc.SimpleMessage{}
 
527
 
 
528
        if target == "databaseMaster" || target == "controller" {
 
529
                outLogs = append(outLogs, jc.SimpleMessage{
 
530
                        loggo.INFO, "waiting for other controllers to be ready for upgrade",
 
531
                })
 
532
                var waitMsg string
 
533
                switch target {
 
534
                case "databaseMaster":
 
535
                        waitMsg = "all controllers are ready to run upgrade steps"
 
536
                case "controller":
 
537
                        waitMsg = "the master has completed its upgrade steps"
 
538
                }
 
539
                outLogs = append(outLogs, jc.SimpleMessage{loggo.INFO, "finished waiting - " + waitMsg})
 
540
        }
 
541
 
 
542
        outLogs = append(outLogs, jc.SimpleMessage{
 
543
                loggo.INFO, fmt.Sprintf(
 
544
                        `starting upgrade from %s to %s for "machine-0"`,
 
545
                        s.oldVersion.Number, jujuversion.Current),
 
546
        })
 
547
 
 
548
        failMessage := fmt.Sprintf(
 
549
                `upgrade from %s to %s for "machine-0" failed \(%%s\): %s`,
 
550
                s.oldVersion.Number, jujuversion.Current, failReason)
 
551
 
 
552
        for i := 0; i < retryCount; i++ {
 
553
                outLogs = append(outLogs, jc.SimpleMessage{loggo.ERROR, fmt.Sprintf(failMessage, "will retry")})
 
554
        }
 
555
        if expectFail {
 
556
                outLogs = append(outLogs, jc.SimpleMessage{loggo.ERROR, fmt.Sprintf(failMessage, "giving up")})
 
557
        } else {
 
558
                outLogs = append(outLogs, jc.SimpleMessage{loggo.INFO,
 
559
                        fmt.Sprintf(`upgrade to %s completed successfully.`, jujuversion.Current)})
 
560
        }
 
561
        return outLogs
 
562
}
 
563
 
 
564
func (s *UpgradeSuite) assertEnvironAgentVersion(c *gc.C, expected version.Number) {
 
565
        envConfig, err := s.State.ModelConfig()
 
566
        c.Assert(err, jc.ErrorIsNil)
 
567
        agentVersion, ok := envConfig.AgentVersion()
 
568
        c.Assert(ok, jc.IsTrue)
 
569
        c.Assert(agentVersion, gc.Equals, expected)
 
570
}
 
571
 
 
572
// NewFakeConfigSetter returns a fakeConfigSetter which implements
 
573
// just enough of the agent.ConfigSetter interface to keep the upgrade
 
574
// steps worker happy.
 
575
func NewFakeConfigSetter(agentTag names.Tag, initialVersion version.Number) *fakeConfigSetter {
 
576
        return &fakeConfigSetter{
 
577
                AgentTag: agentTag,
 
578
                Version:  initialVersion,
 
579
        }
 
580
}
 
581
 
 
582
type fakeConfigSetter struct {
 
583
        agent.ConfigSetter
 
584
        AgentTag names.Tag
 
585
        Version  version.Number
 
586
}
 
587
 
 
588
func (s *fakeConfigSetter) Tag() names.Tag {
 
589
        return s.AgentTag
 
590
}
 
591
 
 
592
func (s *fakeConfigSetter) UpgradedToVersion() version.Number {
 
593
        return s.Version
 
594
}
 
595
 
 
596
func (s *fakeConfigSetter) SetUpgradedToVersion(newVersion version.Number) {
 
597
        s.Version = newVersion
 
598
}
 
599
 
 
600
// NewFakeAgent returns a fakeAgent which implements the agent.Agent
 
601
// interface. This provides enough MachineAgent functionality to
 
602
// support upgrades.
 
603
func NewFakeAgent(confSetter agent.ConfigSetter) *fakeAgent {
 
604
        return &fakeAgent{
 
605
                config: confSetter,
 
606
        }
 
607
}
 
608
 
 
609
type fakeAgent struct {
 
610
        config agent.ConfigSetter
 
611
}
 
612
 
 
613
func (a *fakeAgent) CurrentConfig() agent.Config {
 
614
        return a.config
 
615
}
 
616
 
 
617
func (a *fakeAgent) ChangeConfig(mutate agent.ConfigMutator) error {
 
618
        return mutate(a.config)
 
619
}
 
620
 
 
621
type StatusCall struct {
 
622
        Status status.Status
 
623
        Info   string
 
624
}
 
625
 
 
626
type testStatusSetter struct {
 
627
        Calls []StatusCall
 
628
}
 
629
 
 
630
func (s *testStatusSetter) SetStatus(status status.Status, info string, _ map[string]interface{}) error {
 
631
        s.Calls = append(s.Calls, StatusCall{status, info})
 
632
        return nil
 
633
}