1
# -*- test-case-name: buildbot.test.test_step -*-
4
from StringIO import StringIO
6
from twisted.python import log, components
7
from twisted.python.failure import Failure
8
from twisted.internet import reactor, defer, error
9
from twisted.spread import pb
11
from buildbot import interfaces
12
from buildbot.twcompat import implements
13
from buildbot.util import now
14
from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
15
from buildbot.status.builder import Results, BuildRequestStatus
16
from buildbot.status.progress import BuildProgress
19
"""I represent a request to a specific Builder to run a single build.
21
I have a SourceStamp which specifies what sources I will build. This may
22
specify a specific revision of the source tree (so source.branch,
23
source.revision, and source.patch are used). The .patch attribute is
24
either None or a tuple of (patchlevel, diff), consisting of a number to
25
use in 'patch -pN', and a unified-format context diff.
27
Alternatively, the SourceStamp may specify a set of Changes to be built,
28
contained in source.changes. In this case, I may be mergeable with other
29
BuildRequests on the same branch.
31
I may be part of a BuildSet, in which case I will report status results
34
I am paired with a BuildRequestStatus object, to which I feed status
37
@type source: a L{buildbot.buildset.SourceStamp} instance.
38
@ivar source: the source code that this BuildRequest use
41
@ivar reason: the reason this Build is being requested. Schedulers
42
provide this, but for forced builds the user requesting the
43
build will provide a string.
45
@ivar status: the IBuildStatus object which tracks our status
47
@ivar submittedAt: a timestamp (seconds since epoch) when this request
48
was submitted to the Builder. This is used by the CVS
49
step to compute a checkout timestamp.
54
startCount = 0 # how many times we have tried to start this build
57
implements(interfaces.IBuildRequestControl)
59
__implements__ = interfaces.IBuildRequestControl,
61
def __init__(self, reason, source, builderName=None):
62
# TODO: remove the =None on builderName, it is there so I don't have
63
# to change a lot of tests that create BuildRequest objects
64
assert interfaces.ISourceStamp(source, None)
67
self.start_watchers = []
68
self.finish_watchers = []
69
self.status = BuildRequestStatus(source, builderName)
71
def canBeMergedWith(self, other):
72
return self.source.canBeMergedWith(other.source)
74
def mergeWith(self, others):
75
return self.source.mergeWith([o.source for o in others])
77
def mergeReasons(self, others):
78
"""Return a reason for the merged build request."""
80
for req in [self] + others:
81
if req.reason and req.reason not in reasons:
82
reasons.append(req.reason)
83
return ", ".join(reasons)
85
def waitUntilFinished(self):
86
"""Get a Deferred that will fire (with a
87
L{buildbot.interfaces.IBuildStatus} instance when the build
90
self.finish_watchers.append(d)
93
# these are called by the Builder
95
def requestSubmitted(self, builder):
96
# the request has been placed on the queue
97
self.builder = builder
99
def buildStarted(self, build, buildstatus):
100
"""This is called by the Builder when a Build has been started in the
101
hopes of satifying this BuildRequest. It may be called multiple
102
times, since interrupted builds and lost buildslaves may force
103
multiple Builds to be run until the fate of the BuildRequest is known
105
for o in self.start_watchers[:]:
106
# these observers get the IBuildControl
108
# while these get the IBuildStatus
109
self.status.buildStarted(buildstatus)
111
def finished(self, buildstatus):
112
"""This is called by the Builder when the BuildRequest has been
113
retired. This happens when its Build has either succeeded (yay!) or
114
failed (boo!). TODO: If it is halted due to an exception (oops!), or
115
some other retryable error, C{finished} will not be called yet."""
117
for w in self.finish_watchers:
118
w.callback(buildstatus)
119
self.finish_watchers = []
121
# IBuildRequestControl
123
def subscribe(self, observer):
124
self.start_watchers.append(observer)
125
def unsubscribe(self, observer):
126
self.start_watchers.remove(observer)
129
"""Cancel this request. This can only be successful if the Build has
130
not yet been started.
132
@return: a boolean indicating if the cancel was successful."""
134
return self.builder.cancelBuildRequest(self)
139
"""I represent a single build by a single bot. Specialized Builders can
140
use subclasses of Build to hold status information unique to those build
143
I control B{how} the build proceeds. The actual build is broken up into a
144
series of steps, saved in the .buildSteps[] array as a list of
145
L{buildbot.process.step.BuildStep} objects. Each step is a single remote
146
command, possibly a shell command.
148
During the build, I put status information into my C{BuildStatus}
151
After the build, I go away.
153
I can be used by a factory by setting buildClass on
154
L{buildbot.process.factory.BuildFactory}
156
@ivar request: the L{BuildRequest} that triggered me
157
@ivar build_status: the L{buildbot.status.builder.BuildStatus} that
161
implements(interfaces.IBuildControl)
163
__implements__ = interfaces.IBuildControl,
171
def __init__(self, requests):
172
self.requests = requests
173
for req in self.requests:
176
# build a source stamp
177
self.source = requests[0].mergeWith(requests[1:])
178
self.reason = requests[0].mergeReasons(requests[1:])
180
#self.abandoned = False
183
self.currentStep = None
184
self.slaveEnvironment = {}
186
def setBuilder(self, builder):
188
Set the given builder as our builder.
190
@type builder: L{buildbot.process.builder.Builder}
192
self.builder = builder
194
def setLocks(self, locks):
197
def getSourceStamp(self):
200
def allChanges(self):
201
return self.source.changes
204
# return a list of all source files that were changed
207
for c in self.allChanges():
215
return "<Build %s>" % (self.builder.name,)
217
def __getstate__(self):
218
d = self.__dict__.copy()
219
if d.has_key('remote'):
225
for c in self.allChanges():
226
if c.who not in blamelist:
227
blamelist.append(c.who)
231
def changesText(self):
233
for c in self.allChanges():
234
changetext += "-" * 60 + "\n\n" + c.asText() + "\n"
235
# consider sorting these by number
238
def setSteps(self, steps):
239
"""Set a list of StepFactories, which are generally just class
240
objects which derive from step.BuildStep . These are used to create
241
the Steps themselves when the Build starts (as opposed to when it is
242
first created). By creating the steps later, their __init__ method
243
will have access to things like build.allFiles() ."""
244
self.stepFactories = steps # tuples of (factory, kwargs)
253
def getSlaveCommandVersion(self, command, oldversion=None):
254
return self.slavebuilder.getSlaveCommandVersion(command, oldversion)
256
def startBuild(self, build_status, expectations, slavebuilder):
257
"""This method sets up the build, then starts it by invoking the
258
first Step. It returns a Deferred which will fire when the build
259
finishes. This Deferred is guaranteed to never errback."""
261
# we are taking responsibility for watching the connection to the
262
# remote. This responsibility was held by the Builder until our
263
# startBuild was called, and will not return to them until we fire
264
# the Deferred returned by this method.
266
log.msg("%s.startBuild" % self)
267
self.build_status = build_status
268
self.slavebuilder = slavebuilder
269
self.slavename = slavebuilder.slave.slavename
270
# convert all locks into their real forms
271
self.locks = [self.builder.botmaster.getLockByID(l)
273
# then narrow SlaveLocks down to the right slave
274
self.locks = [l.getLock(self.slavebuilder) for l in self.locks]
275
self.remote = slavebuilder.remote
276
self.remote.notifyOnDisconnect(self.lostRemote)
277
d = self.deferred = defer.Deferred()
280
self.setupBuild(expectations) # create .steps
282
# the build hasn't started yet, so log the exception as a point
283
# event instead of flunking the build. TODO: associate this
284
# failure with the build instead. this involves doing
285
# self.build_status.buildStarted() from within the exception
287
log.msg("Build.setupBuild failed")
289
self.builder.builder_status.addPointEvent(["setupBuild",
293
self.results = FAILURE
298
self.build_status.buildStarted(self)
299
self.acquireLocks().addCallback(self._startBuild_2)
302
def acquireLocks(self, res=None):
303
log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks))
305
return defer.succeed(None)
306
for lock in self.locks:
307
if not lock.isAvailable():
308
log.msg("Build %s waiting for lock %s" % (self, lock))
309
d = lock.waitUntilAvailable(self)
310
d.addCallback(self.acquireLocks)
312
# all locks are available, claim them all
313
for lock in self.locks:
315
return defer.succeed(None)
317
def _startBuild_2(self, res):
320
def setupBuild(self, expectations):
321
# create the actual BuildSteps. If there are any name collisions, we
322
# add a count to the loser until it is unique.
324
self.stepStatuses = {}
328
for factory, args in self.stepFactories:
330
if not args.has_key("workdir"):
331
args['workdir'] = self.workdir
332
step = factory(build=self, **args)
335
while name in stepnames and count < 100:
337
name = step.name + "_%d" % count
338
if name in stepnames:
339
raise RuntimeError("duplicate step '%s'" % step.name)
341
stepnames.append(name)
342
self.steps.append(step)
344
# tell the BuildStatus about the step. This will create a
345
# BuildStepStatus and bind it to the Step.
346
self.build_status.addStep(step)
350
# XXX: maybe bail if step.progressMetrics is empty? or skip
351
# progress for that one step (i.e. "it is fast"), or have a
352
# separate "variable" flag that makes us bail on progress
354
sp = step.setupProgress()
358
# Create a buildbot.status.progress.BuildProgress object. This is
359
# called once at startup to figure out how to build the long-term
360
# Expectations object, and again at the start of each build to get a
361
# fresh BuildProgress object to track progress for that individual
362
# build. TODO: revisit at-startup call
365
self.progress = BuildProgress(sps)
366
if self.progress and expectations:
367
self.progress.setExpectationsFrom(expectations)
369
# we are now ready to set up our BuildStatus.
370
self.build_status.setSourceStamp(self.source)
371
self.build_status.setReason(self.reason)
372
self.build_status.setBlamelist(self.blamelist())
373
self.build_status.setProgress(self.progress)
375
self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED
376
self.result = SUCCESS # overall result, may downgrade after each step
377
self.text = [] # list of text string lists (text2)
379
def getNextStep(self):
380
"""This method is called to obtain the next BuildStep for this build.
381
When it returns None (or raises a StopIteration exception), the build
385
return self.steps.pop(0)
387
def startNextStep(self):
389
s = self.getNextStep()
390
except StopIteration:
393
return self.allStepsDone()
395
d = defer.maybeDeferred(s.startStep, self.remote)
396
d.addCallback(self._stepDone, s)
397
d.addErrback(self.buildException)
399
def _stepDone(self, results, step):
400
self.currentStep = None
402
return # build was interrupted, don't keep building
403
terminate = self.stepDone(results, step) # interpret/merge results
405
return self.allStepsDone()
408
def stepDone(self, result, step):
409
"""This method is called when the BuildStep completes. It is passed a
410
status object from the BuildStep and is responsible for merging the
411
Step's results into those of the overall Build."""
415
if type(result) == types.TupleType:
416
result, text = result
417
assert type(result) == type(SUCCESS)
418
log.msg(" step '%s' complete: %s" % (step.name, Results[result]))
419
self.results.append(result)
421
self.text.extend(text)
424
if result == FAILURE:
425
if step.warnOnFailure:
426
if self.result != FAILURE:
427
self.result = WARNINGS
428
if step.flunkOnFailure:
429
self.result = FAILURE
430
if step.haltOnFailure:
431
self.result = FAILURE
433
elif result == WARNINGS:
434
if step.warnOnWarnings:
435
if self.result != FAILURE:
436
self.result = WARNINGS
437
if step.flunkOnWarnings:
438
self.result = FAILURE
439
elif result == EXCEPTION:
440
self.result = EXCEPTION
444
def lostRemote(self, remote=None):
445
# the slave went away. There are several possible reasons for this,
446
# and they aren't necessarily fatal. For now, kill the build, but
447
# TODO: see if we can resume the build when it reconnects.
448
log.msg("%s.lostRemote" % self)
451
# this should cause the step to finish.
452
log.msg(" stopping currentStep", self.currentStep)
453
self.currentStep.interrupt(Failure(error.ConnectionLost()))
455
def stopBuild(self, reason="<no reason given>"):
456
# the idea here is to let the user cancel a build because, e.g.,
457
# they realized they committed a bug and they don't want to waste
458
# the time building something that they know will fail. Another
459
# reason might be to abandon a stuck build. We want to mark the
460
# build as failed quickly rather than waiting for the slave's
461
# timeout to kill it on its own.
463
log.msg(" %s: stopping build: %s" % (self, reason))
466
# TODO: include 'reason' in this point event
467
self.builder.builder_status.addPointEvent(['interrupt'])
468
self.currentStep.interrupt(reason)
470
# TODO: maybe let its deferred do buildFinished
471
if self.currentStep and self.currentStep.progress:
472
# XXX: really .fail or something
473
self.currentStep.progress.finish()
474
text = ["stopped", reason]
475
self.buildFinished(text, "red", FAILURE)
477
def allStepsDone(self):
478
if self.result == FAILURE:
481
elif self.result == WARNINGS:
484
elif self.result == EXCEPTION:
489
text = ["build", "successful"]
490
text.extend(self.text)
491
return self.buildFinished(text, color, self.result)
493
def buildException(self, why):
494
log.msg("%s.buildException" % self)
496
self.buildFinished(["build", "exception"], "purple", FAILURE)
498
def buildFinished(self, text, color, results):
499
"""This method must be called when the last Step has completed. It
500
marks the Build as complete and returns the Builder to the 'idle'
503
It takes three arguments which describe the overall build status:
504
text, color, results. 'results' is one of SUCCESS, WARNINGS, or
507
If 'results' is SUCCESS or WARNINGS, we will permit any dependant
508
builds to start. If it is 'FAILURE', those builds will be
513
self.remote.dontNotifyOnDisconnect(self.lostRemote)
514
self.results = results
516
log.msg(" %s: build finished" % self)
517
self.build_status.setSlavename(self.slavename)
518
self.build_status.setText(text)
519
self.build_status.setColor(color)
520
self.build_status.setResults(results)
521
self.build_status.buildFinished()
523
# XXX: also test a 'timing consistent' flag?
524
log.msg(" setting expectations for next time")
525
self.builder.setExpectations(self.progress)
526
reactor.callLater(0, self.releaseLocks)
527
self.deferred.callback(self)
530
def releaseLocks(self):
531
log.msg("releaseLocks(%s): %s" % (self, self.locks))
532
for lock in self.locks:
538
return self.build_status
540
# stopBuild is defined earlier
added
removed
buildbot/process/base.py
