2
# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 The SCons Foundation
4
# Permission is hereby granted, free of charge, to any person obtaining
5
# a copy of this software and associated documentation files (the
6
# "Software"), to deal in the Software without restriction, including
7
# without limitation the rights to use, copy, modify, merge, publish,
8
# distribute, sublicense, and/or sell copies of the Software, and to
9
# permit persons to whom the Software is furnished to do so, subject to
10
# the following conditions:
12
# The above copyright notice and this permission notice shall be included
13
# in all copies or substantial portions of the Software.
15
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
16
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
17
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
Generic Taskmaster module for the SCons build engine.
27
This module contains the primary interface(s) between a wrapping user
28
interface and the SCons build engine. There are two key classes here:
31
This is the main engine for walking the dependency graph and
32
calling things to decide what does or doesn't need to be built.
35
This is the base class for allowing a wrapping interface to
36
decide what does or doesn't actually need to be done. The
37
intention is for a wrapping interface to subclass this as
38
appropriate for different types of behavior it may need.
40
The canonical example is the SCons native Python interface,
41
which has Task subclasses that handle its specific behavior,
42
like printing "`foo' is up to date" when a top-level target
43
doesn't need to be built, and handling the -c option by removing
44
targets as its "build" action. There is also a separate subclass
45
for suppressing this output when the -q option is used.
47
The Taskmaster instantiates a Task object for each (set of)
48
target(s) that it decides need to be evaluated and/or built.
51
__revision__ = "src/engine/SCons/Taskmaster.py 2523 2007/12/12 09:37:41 knight"
63
StateString = SCons.Node.StateString
67
# A subsystem for recording stats about how different Nodes are handled by
68
# the main Taskmaster loop. There's no external control here (no need for
69
# a --debug= option); enable it by changing the value of CollectStats.
75
A simple class for holding statistics about the disposition of a
76
Node by the Taskmaster. If we're collecting statistics, each Node
77
processed by the Taskmaster gets one of these attached, in which case
78
the Taskmaster records its decision each time it processes the Node.
79
(Ideally, that's just once per Node.)
83
Instantiates a Taskmaster.Stats object, initializing all
84
appropriate counters to zero.
87
self.already_handled = 0
96
fmt = "%(considered)3d "\
97
"%(already_handled)3d " \
99
"%(child_failed)3d " \
101
"%(side_effects)3d " \
105
StatsNodes.sort(lambda a, b: cmp(str(a), str(b)))
107
print (fmt % n.stats.__dict__) + str(n)
113
Default SCons build engine task.
115
This controls the interaction of the actual building of node
116
and the rest of the engine.
118
This is expected to handle all of the normally-customizable
119
aspects of controlling a build, so any given application
120
*should* be able to do what it wants by sub-classing this
121
class and overriding methods as appropriate. If an application
122
needs to customze something by sub-classing Taskmaster (or
123
some other build engine class), we should first try to migrate
124
that functionality into this class.
126
Note that it's generally a good idea for sub-classes to call
127
these methods explicitly to update state, etc., rather than
128
roll their own interaction with Taskmaster from scratch.
130
def __init__(self, tm, targets, top, node):
132
self.targets = targets
137
def display(self, message):
139
Hook to allow the calling interface to display a message.
141
This hook gets called as part of preparing a task for execution
142
(that is, a Node to be built). As part of figuring out what Node
143
should be built next, the actually target list may be altered,
144
along with a message describing the alteration. The calling
145
interface can subclass Task and provide a concrete implementation
146
of this method to see those messages.
152
Called just before the task is executed.
154
This is mainly intended to give the target Nodes a chance to
155
unlink underlying files and make all necessary directories before
156
the Action is actually called to build the targets.
159
# Now that it's the appropriate time, give the TaskMaster a
160
# chance to raise any exceptions it encountered while preparing
162
self.exception_raise()
165
self.display(self.tm.message)
166
self.tm.message = None
168
for t in self.targets:
170
for s in t.side_effects:
173
def get_target(self):
174
"""Fetch the target being built or updated by this task.
180
Called to execute the task.
182
This method is called from multiple threads in a parallel build,
183
so only do thread safe stuff here. Do thread unsafe stuff in
184
prepare(), executed() or failed().
188
everything_was_cached = 1
189
for t in self.targets:
190
if not t.retrieve_from_cache():
191
everything_was_cached = 0
193
if not everything_was_cached:
194
self.targets[0].build()
195
except KeyboardInterrupt:
198
exc_value = sys.exc_info()[1]
199
raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code)
200
except SCons.Errors.UserError:
202
except SCons.Errors.BuildError:
205
raise SCons.Errors.TaskmasterException(self.targets[0],
208
def executed_without_callbacks(self):
210
Called when the task has been successfully executed
211
and the Taskmaster instance doesn't want to call
212
the Node's callback methods.
214
for t in self.targets:
215
if t.get_state() == SCons.Node.executing:
216
for side_effect in t.side_effects:
217
side_effect.set_state(SCons.Node.no_state)
218
t.set_state(SCons.Node.executed)
220
def executed_with_callbacks(self):
222
Called when the task has been successfully executed and
223
the Taskmaster instance wants to call the Node's callback
226
This may have been a do-nothing operation (to preserve build
227
order), so we must check the node's state before deciding whether
228
it was "built", in which case we call the appropriate Node method.
229
In any event, we always call "visited()", which will handle any
230
post-visit actions that must take place regardless of whether
231
or not the target was an actual built target or a source Node.
233
for t in self.targets:
234
if t.get_state() == SCons.Node.executing:
235
for side_effect in t.side_effects:
236
side_effect.set_state(SCons.Node.no_state)
237
t.set_state(SCons.Node.executed)
241
executed = executed_with_callbacks
245
Default action when a task fails: stop the build.
251
Explicit stop-the-build failure.
253
for t in self.targets:
254
t.set_state(SCons.Node.failed)
257
# We're stopping because of a build failure, but give the
258
# calling Task class a chance to postprocess() the top-level
259
# target under which the build failure occurred.
260
self.targets = [self.tm.current_top]
263
def fail_continue(self):
265
Explicit continue-the-build failure.
267
This sets failure status on the target nodes and all of
268
their dependent parent nodes.
270
for t in self.targets:
271
# Set failure state on all of the parents that were dependent
272
# on this failed build.
273
def set_state(node): node.set_state(SCons.Node.failed)
274
t.call_for_all_waiting_parents(set_state)
276
def make_ready_all(self):
278
Marks all targets in a task ready for execution.
280
This is used when the interface needs every target Node to be
281
visited--the canonical example being the "scons -c" option.
283
self.out_of_date = self.targets[:]
284
for t in self.targets:
285
t.disambiguate().set_state(SCons.Node.executing)
286
for s in t.side_effects:
287
s.set_state(SCons.Node.executing)
289
def make_ready_current(self):
291
Marks all targets in a task ready for execution if any target
294
This is the default behavior for building only what's necessary.
296
self.out_of_date = []
297
for t in self.targets:
299
t.disambiguate().make_ready()
300
is_up_to_date = not t.has_builder() or \
301
(not t.always_build and t.is_up_to_date())
302
except EnvironmentError, e:
303
raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filename=e.filename)
305
t.set_state(SCons.Node.up_to_date)
307
self.out_of_date.append(t)
308
t.set_state(SCons.Node.executing)
309
for s in t.side_effects:
310
s.set_state(SCons.Node.executing)
312
make_ready = make_ready_current
314
def postprocess(self):
316
Post-processes a task after it's been executed.
318
This examines all the targets just built (or not, we don't care
319
if the build was successful, or even if there was no build
320
because everything was up-to-date) to see if they have any
321
waiting parent Nodes, or Nodes waiting on a common side effect,
322
that can be put back on the candidates list.
325
# We may have built multiple targets, some of which may have
326
# common parents waiting for this build. Count up how many
327
# targets each parent was waiting for so we can subtract the
328
# values later, and so we *don't* put waiting side-effect Nodes
329
# back on the candidates list if the Node is also a waiting
332
targets = set(self.targets)
336
for p in t.waiting_parents.keys():
337
parents[p] = parents.get(p, 0) + 1
340
for s in t.side_effects:
341
if s.get_state() == SCons.Node.executing:
342
s.set_state(SCons.Node.no_state)
343
for p in s.waiting_parents.keys():
344
if not parents.has_key(p):
346
for p in s.waiting_s_e.keys():
348
self.tm.candidates.append(p)
350
for p, subtract in parents.items():
351
p.ref_count = p.ref_count - subtract
353
self.tm.candidates.append(p)
358
# Exception handling subsystem.
360
# Exceptions that occur while walking the DAG or examining Nodes
361
# must be raised, but must be raised at an appropriate time and in
362
# a controlled manner so we can, if necessary, recover gracefully,
363
# possibly write out signature information for Nodes we've updated,
364
# etc. This is done by having the Taskmaster tell us about the
365
# exception, and letting
369
Returns info about a recorded exception.
371
return self.exception
375
Clears any recorded exception.
377
This also changes the "exception_raise" attribute to point
378
to the appropriate do-nothing method.
380
self.exception = (None, None, None)
381
self.exception_raise = self._no_exception_to_raise
383
def exception_set(self, exception=None):
385
Records an exception to be raised at the appropriate time.
387
This also changes the "exception_raise" attribute to point
388
to the method that will, in fact
391
exception = sys.exc_info()
392
self.exception = exception
393
self.exception_raise = self._exception_raise
395
def _no_exception_to_raise(self):
398
def _exception_raise(self):
400
Raises a pending exception that was recorded while getting a
401
Task ready for execution.
403
exc = self.exc_info()[:]
405
exc_type, exc_value, exc_traceback = exc
407
exc_type, exc_value = exc
409
raise exc_type, exc_value, exc_traceback
412
def find_cycle(stack):
413
if stack[0] == stack[-1]:
415
for n in stack[-1].waiting_parents.keys():
417
if find_cycle(stack):
425
The Taskmaster for walking the dependency DAG.
428
def __init__(self, targets=[], tasker=Task, order=None, trace=None):
429
self.original_top = targets
430
self.top_targets_left = targets[:]
431
self.top_targets_left.reverse()
439
self.next_candidate = self.find_next_candidate
441
def find_next_candidate(self):
443
Returns the next candidate Node for (potential) evaluation.
445
The candidate list (really a stack) initially consists of all of
446
the top-level (command line) targets provided when the Taskmaster
447
was initialized. While we walk the DAG, visiting Nodes, all the
448
children that haven't finished processing get pushed on to the
449
candidate list. Each child can then be popped and examined in
450
turn for whether *their* children are all up-to-date, in which
451
case a Task will be created for their actual evaluation and
454
Here is where we also allow candidate Nodes to alter the list of
455
Nodes that should be examined. This is used, for example, when
456
invoking SCons in a source directory. A source directory Node can
457
return its corresponding build directory Node, essentially saying,
458
"Hey, you really need to build this thing over here instead."
461
return self.candidates.pop()
465
node = self.top_targets_left.pop()
468
self.current_top = node
469
alt, message = node.alter_targets()
471
self.message = message
472
self.candidates.append(node)
473
self.candidates.extend(self.order(alt))
474
node = self.candidates.pop()
477
def no_next_candidate(self):
479
Stops Taskmaster processing by not returning a next candidate.
483
def _find_next_ready_node(self):
485
Finds the next node that is ready to be built.
487
This is *the* main guts of the DAG walk. We loop through the
488
list of candidates, looking for something that has no un-built
489
children (i.e., that is a leaf Node or has dependencies that are
490
all leaf Nodes or up-to-date). Candidate Nodes are re-scanned
491
(both the target Node itself and its sources, which are always
492
scanned in the context of a given target) to discover implicit
493
dependencies. A Node that must wait for some children to be
494
built will be put back on the candidates list after the children
495
have finished building. A Node that has been put back on the
496
candidates list in this way may have itself (or its sources)
497
re-scanned, in order to handle generated header files (e.g.) and
498
the implicit dependencies therein.
500
Note that this method does not do any signature calculation or
501
up-to-date check itself. All of that is handled by the Task
502
class. This is purely concerned with the dependency graph walk.
505
self.ready_exc = None
510
node = self.next_candidate()
514
node = node.disambiguate()
515
state = node.get_state()
518
if not hasattr(node, 'stats'):
520
StatsNodes.append(node)
522
S.considered = S.considered + 1
526
if T: T.write('Taskmaster: %s:' % repr(str(node)))
528
# Skip this node if it has already been evaluated:
529
if state > SCons.Node.pending:
530
if S: S.already_handled = S.already_handled + 1
531
if T: T.write(' already handled (%s)\n' % StateString[state])
534
# Mark this node as being on the execution stack:
535
node.set_state(SCons.Node.pending)
538
children = node.children() + node.prerequisites
540
exc_value = sys.exc_info()[1]
541
e = SCons.Errors.ExplicitExit(node, exc_value.code)
542
self.ready_exc = (SCons.Errors.ExplicitExit, e)
543
if T: T.write(' SystemExit\n')
545
except KeyboardInterrupt:
546
if T: T.write(' KeyboardInterrupt\n')
549
# We had a problem just trying to figure out the
550
# children (like a child couldn't be linked in to a
551
# BuildDir, or a Scanner threw something). Arrange to
552
# raise the exception when the Task is "executed."
553
self.ready_exc = sys.exc_info()
554
if S: S.problem = S.problem + 1
555
if T: T.write(' exception\n')
559
c = map(str, children)
561
T.write(' children:\n %s\n ' % c)
563
childstate = map(lambda N: (N, N.get_state()), children)
565
# Skip this node if any of its children have failed. This
566
# catches the case where we're descending a top-level target
567
# and one of our children failed while trying to be built
568
# by a *previous* descent of an earlier top-level target.
569
failed_children = filter(lambda I: I[1] == SCons.Node.failed,
572
node.set_state(SCons.Node.failed)
573
if S: S.child_failed = S.child_failed + 1
575
c = map(str, failed_children)
577
T.write(' children failed:\n %s\n' % c)
580
# Detect dependency cycles:
581
pending_nodes = filter(lambda I: I[1] == SCons.Node.pending, childstate)
583
for p in pending_nodes:
584
cycle = find_cycle([p[0], node])
586
desc = "Dependency cycle: " + string.join(map(str, cycle), " -> ")
587
if T: T.write(' dependency cycle\n')
588
raise SCons.Errors.UserError, desc
590
not_built = filter(lambda I: I[1] <= SCons.Node.executing, childstate)
592
# We're waiting on one or more derived targets that have
593
# not yet finished building.
595
not_visited = filter(lambda I: not I[1], not_built)
597
# Some of them haven't even been visited yet.
598
# Add them to the list so that on some next pass
599
# we can take a stab at evaluating them (or
601
not_visited = map(lambda I: I[0], not_visited)
602
not_visited.reverse()
603
self.candidates.extend(self.order(not_visited))
605
n_b_nodes = map(lambda I: I[0], not_built)
607
# Add this node to the waiting parents lists of anything
608
# we're waiting on, with a reference count so we can be
609
# put back on the list for re-evaluation when they've
611
map(lambda n, P=node: n.add_to_waiting_parents(P), n_b_nodes)
612
node.ref_count = len(set(n_b_nodes))
614
if S: S.not_built = S.not_built + 1
616
c = map(str, n_b_nodes)
618
T.write(' waiting on unfinished children:\n %s\n' % c)
621
# Skip this node if it has side-effects that are
622
# currently being built:
623
side_effects = filter(lambda N:
624
N.get_state() == SCons.Node.executing,
627
map(lambda n, P=node: n.add_to_waiting_s_e(P), side_effects)
628
if S: S.side_effects = S.side_effects + 1
630
c = map(str, side_effects)
632
T.write(' waiting on side effects:\n %s\n' % c)
635
# The default when we've gotten through all of the checks above:
636
# this node is ready to be built.
637
if S: S.build = S.build + 1
638
if T: T.write(' evaluating %s\n' % node)
645
Returns the next task to be executed.
647
This simply asks for the next Node to be evaluated, and then wraps
648
it in the specific Task subclass with which we were initialized.
650
node = self._find_next_ready_node()
655
tlist = node.get_executor().targets
657
task = self.tasker(self, tlist, node in self.original_top, node)
660
except KeyboardInterrupt:
663
# We had a problem just trying to get this task ready (like
664
# a child couldn't be linked in to a BuildDir when deciding
665
# whether this node is current). Arrange to raise the
666
# exception when the Task is "executed."
667
self.ready_exc = sys.exc_info()
670
task.exception_set(self.ready_exc)
672
self.ready_exc = None
678
Stops the current build completely.
680
self.next_candidate = self.no_next_candidate
added
removed
scons/scons-local-0.97.0d20071212/SCons/Taskmaster.py
