3
# Thomas Nagy, 2005 (ita)
5
"Atomic operations that create nodes or execute commands"
7
import os, types, shutil
8
try: from hashlib import md5
9
except ImportError: from md5 import md5
11
import Params, Scan, Action, Runner, Object
12
from Params import debug, error, warning
15
"tasks that have been run, this is used in tests to check which tasks were actually launched"
17
g_default_param = {'path_lst':[]}
18
"the default scanner parameter"
21
"""There is a single instance of TaskManager held by Task.py:g_tasks
22
The manager holds a list of TaskGroup
23
Each TaskGroup contains a map(priority, list of tasks)"""
3
# Thomas Nagy, 2005-2008 (ita)
6
Running tasks in parallel is a simple problem, but in practice it is more complicated:
7
* dependencies discovered during the build (dynamic task creation)
8
* dependencies discovered after files are compiled
9
* the amount of tasks and dependencies (graph size) can be huge
11
This is why the dependency management is split on three different levels:
12
1. groups of tasks that run all after another group of tasks
13
2. groups of tasks that can be run in parallel
14
3. tasks that can run in parallel, but with possible unknown ad-hoc dependencies
16
The point #1 represents a strict sequential order between groups of tasks, for example a compiler is produced
17
and used to compile the rest, whereas #2 and #3 represent partial order constraints where #2 applies to the kind of task
18
and #3 applies to the task instances.
20
#1 is held by the task manager: ordered list of TaskGroups (see bld.add_group)
21
#2 is held by the task groups and the task types: precedence after/before (topological sort),
22
and the constraints extracted from file extensions
23
#3 is held by the tasks individually (attribute run_after),
24
and the scheduler (Runner.py) use Task::runnable_status to reorder the tasks
28
To try, use something like this in your code:
29
import Constants, Task
30
Task.algotype = Constants.MAXPARALLEL
34
There are two concepts with the tasks (individual units of change):
35
* dependency (if 1 is recompiled, recompile 2)
36
* order (run 2 after 1)
38
example 1: if t1 depends on t2 and t2 depends on t3 it is not necessary to make t1 depend on t3 (dependency is transitive)
39
example 2: if t1 depends on a node produced by t2, it is not immediately obvious that t1 must run after t2 (order is not obvious)
41
The role of the Task Manager is to give the tasks in order (groups of task that may be run in parallel one after the other)
45
import os, types, shutil, sys, re, random, time
47
import Build, Runner, Utils, Node, Logs, Options
48
from Logs import debug
49
from Constants import *
52
#algotype = JOBCONTROL
53
#algotype = MAXPARALLEL
56
Enable different kind of dependency algorithms:
57
1 make groups: first compile all cpps and then compile all links (NORMAL)
58
2 parallelize all (each link task run after its dependencies) (MAXPARALLEL)
59
3 like 1 but provide additional constraints for the parallelization (MAXJOBS)
61
In theory 1. will be faster than 2 for waf, but might be slower for builds
62
The scheme 2 will not allow for running tasks one by one so it can cause disk thrashing on huge builds
68
"""Infer dependencies from task input and output nodes
74
(ins, outs) = v[x.env.variant()]
78
v[x.env.variant()] = (ins, outs)
80
for a in getattr(x, 'inputs', []):
81
try: ins[a.id].append(x)
82
except KeyError: ins[a.id] = [x]
83
for a in getattr(x, 'outputs', []):
84
try: outs[a.id].append(x)
85
except KeyError: outs[a.id] = [x]
87
for (ins, outs) in v.values():
88
links = set(ins.iterkeys()).intersection(outs.iterkeys())
94
class TaskManager(object):
95
"""The manager is attached to the build object, it holds a list of TaskGroup"""
24
96
def __init__(self):
27
def add_group(self, name=''):
29
try: size = len(self.groups)
31
name = 'group-%d' % size
33
self.groups = [TaskGroup(name)]
35
if not self.groups[0].prio:
36
warning('add_group: an empty group is already present')
38
self.groups = self.groups + [TaskGroup(name)]
39
def add_task(self, task, prio):
40
if not self.groups: self.add_group('group-0')
43
self.groups[-1].add_task(task, prio)
99
self.current_group = 0
101
def get_next_set(self):
102
"""return the next set of tasks to execute
103
the first parameter is the maximum amount of parallelization that may occur"""
105
while not ret and self.current_group < len(self.groups):
106
ret = self.groups[self.current_group].get_next_set()
108
else: self.current_group += 1
112
if self.groups and not self.groups[0].tasks:
113
warn('add_group: an empty group is already present')
114
self.groups.append(TaskGroup())
116
def add_task(self, task):
117
if not self.groups: self.add_group()
118
self.groups[-1].add_task(task)
46
122
if not self.groups: return 0
47
123
for group in self.groups:
49
total += len(group.prio[p])
124
total += len(group.tasks)
53
print "-----group-------", i.name
55
print "prio: ", j, str(i.prio[j])
57
"the container of all tasks (instance of TaskManager)"
58
g_tasks = TaskManager()
61
"A TaskGroup maps priorities (integers) to lists of tasks"
62
def __init__(self, name):
65
self.prio = {} # map priority numbers to tasks
66
def add_task(self, task, prio):
67
try: self.prio[prio].append(task)
68
except: self.prio[prio] = [task]
71
"TaskBase is the base class for task objects"
72
def __init__(self, priority, normal=1):
77
# add to the list of tasks
78
g_tasks.add_task(self, priority)
127
def add_finished(self, tsk):
128
self.tasks_done.append(tsk)
129
bld = tsk.generator.bld
130
if Options.is_install:
132
if 'install' in tsk.__dict__:
133
f = tsk.__dict__['install']
138
class TaskGroup(object):
139
"the compilation of one group does not begin until the previous group has finished (in the manager)"
141
self.tasks = [] # this list will be consumed
143
self.cstr_groups = {} # tasks having equivalent constraints
144
self.cstr_order = {} # partial order between the cstr groups
145
self.temp_tasks = [] # tasks put on hold
149
"clears the state of the object (put back the tasks into self.tasks)"
150
for x in self.cstr_groups:
151
self.tasks += self.cstr_groups[x]
152
self.tasks = self.temp_tasks + self.tasks
154
self.cstr_groups = []
159
"prepare the scheduling"
162
file_deps(self.tasks)
163
self.make_cstr_groups()
164
self.extract_constraints()
166
def get_next_set(self):
167
"next list of tasks to execute using max job settings, returns (maxjobs, task_list)"
169
if algotype == NORMAL:
170
tasks = self.tasks_in_parallel()
172
elif algotype == JOBCONTROL:
173
(maxj, tasks) = self.tasks_by_max_jobs()
174
elif algotype == MAXPARALLEL:
175
tasks = self.tasks_with_inner_constraints()
80
self.m_idx = g_tasks.idx
83
"return non-zero if the task may is ready"
86
"return 0 if the task does not need to run"
89
"prepare the task for further processing"
91
def update_stat(self):
178
raise Utils.WafError("unknown algorithm type %s" % (algotype))
180
if not tasks: return ()
183
def make_cstr_groups(self):
184
"unite the tasks that have similar constraints"
185
self.cstr_groups = {}
187
h = x.hash_constraints()
188
try: self.cstr_groups[h].append(x)
189
except KeyError: self.cstr_groups[h] = [x]
191
def add_task(self, task):
192
try: self.tasks.append(task)
193
except KeyError: self.tasks = [task]
195
def set_order(self, a, b):
196
try: self.cstr_order[a].add(b)
197
except KeyError: self.cstr_order[a] = set([b,])
199
def compare_exts(self, t1, t2):
200
"extension production"
204
out_ = t2.attr(y, ())
209
out_ = t1.attr(y, ())
215
def compare_partial(self, t1, t2):
216
"partial relations after/before"
219
name = t2.__class__.__name__
220
if name in t1.attr(m, ()): return -1
221
elif name in t1.attr(n, ()): return 1
222
name = t1.__class__.__name__
223
if name in t2.attr(m, ()): return 1
224
elif name in t2.attr(n, ()): return -1
227
def extract_constraints(self):
228
"extract the parallelization constraints from the tasks with different constraints"
229
keys = self.cstr_groups.keys()
231
# hopefully the length of this list is short
232
for i in xrange(max):
233
t1 = self.cstr_groups[keys[i]][0]
234
for j in xrange(i + 1, max):
235
t2 = self.cstr_groups[keys[j]][0]
237
# add the constraints based on the comparisons
238
val = (self.compare_exts(t1, t2)
239
or self.compare_partial(t1, t2)
242
self.set_order(keys[i], keys[j])
244
self.set_order(keys[j], keys[i])
246
# TODO: extract constraints by file extensions on the actions
248
def tasks_in_parallel(self):
249
"(NORMAL) next list of tasks that may be executed in parallel"
251
if not self.ready: self.prepare()
253
keys = self.cstr_groups.keys()
259
for k in self.cstr_order.values():
264
unconnected.append(u)
267
for y in unconnected:
268
toreturn.extend(self.cstr_groups[y])
270
# remove stuff only after
271
for y in unconnected:
272
try: self.cstr_order.__delitem__(y)
273
except KeyError: pass
274
self.cstr_groups.__delitem__(y)
276
if not toreturn and remainder:
277
raise Utils.WafError("circular order constraint detected %r" % remainder)
281
def tasks_by_max_jobs(self):
282
"(JOBCONTROL) returns the tasks that can run in parallel with the max amount of jobs"
283
if not self.ready: self.prepare()
284
if not self.temp_tasks: self.temp_tasks = self.tasks_in_parallel()
285
if not self.temp_tasks: return (None, None)
290
for t in self.temp_tasks:
291
m = getattr(t, "maxjobs", getattr(self.__class__, "maxjobs", MAXJOBS))
300
self.temp_tasks = remaining
301
return (maxjobs, ret)
303
def tasks_with_inner_constraints(self):
304
"""(MAXPARALLEL) returns all tasks in this group, but add the constraints on each task instance
305
as an optimization, it might be desirable to discard the tasks which do not have to run"""
306
if not self.ready: self.prepare()
308
if getattr(self, "done", None): return None
310
for p in self.cstr_order:
311
for v in self.cstr_order[p]:
312
for m in self.cstr_groups[p]:
313
for n in self.cstr_groups[v]:
316
self.cstr_groups = {}
318
return self.tasks[:] # make a copy
320
class store_task_type(type):
321
"store the task types that have a name ending in _task into a map (remember the existing task types)"
322
def __init__(cls, name, bases, dict):
323
super(store_task_type, cls).__init__(name, bases, dict)
326
if name.endswith('_task'):
327
name = name.replace('_task', '')
328
TaskBase.classes[name] = cls
330
class TaskBase(object):
331
"""Base class for all Waf tasks
333
The most important methods are (by usual order of call):
334
1 runnable_status: ask the task if it should be run, skipped, or if we have to ask later
335
2 __str__: string to display to the user
336
3 run: execute the task
337
4 post_run: after the task is run, update the cache about the task
339
This class should be seen as an interface, it provides the very minimum necessary for the scheduler
340
so it does not do much.
342
For illustration purposes, TaskBase instances try to execute self.fun (if provided)
345
__metaclass__ = store_task_type
352
def __init__(self, *k, **kw):
353
self.hasrun = NOT_RUN
356
self.generator = kw['generator']
358
self.generator = self
361
if kw.get('normal', 1):
362
self.generator.bld.task_manager.add_task(self)
366
return '\n\t{task: %s %s}' % (self.__class__.__name__, str(getattr(self, "fun", "")))
369
"string to display to the user"
370
if hasattr(self, 'fun'):
371
return 'executing: %s\n' % self.fun.__name__
372
return self.__class__.__name__ + '\n'
374
def runnable_status(self):
375
"RUN_ME SKIP_ME or ASK_LATER"
378
def can_retrieve_cache(self):
382
if self.can_retrieve_cache():
387
"called if the task must run"
388
if hasattr(self, 'fun'):
389
return self.fun(self)
92
393
"update the dependency tree (node stats)"
98
"prints the debug info"
104
"return the color to use for the console messages"
106
def set_display(self, v):
108
def get_display(self):
109
return self.m_display
397
"print either the description (using __str__) or the progress bar or the ide output"
398
col1 = Logs.colors(self.color)
399
col2 = Logs.colors.NORMAL
401
if Options.options.progress_bar == 1:
402
return self.generator.bld.progress_line(self.position[0], self.position[1], col1, col2)
404
if Options.options.progress_bar == 2:
405
ini = self.generator.bld.ini
406
ela = time.strftime('%H:%M:%S', time.gmtime(time.time() - ini))
408
ins = ','.join([n.name for n in self.inputs])
409
except AttributeError:
412
outs = ','.join([n.name for n in self.outputs])
413
except AttributeError:
415
return '|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n' % (self.position[1], self.position[0], ins, outs, ela)
417
total = self.position[1]
419
fs = '[%%%dd/%%%dd] %%s%%s%%s' % (n, n)
420
return fs % (self.position[0], self.position[1], col1, str(self), col2)
422
def attr(self, att, default=None):
423
"retrieve an attribute from the instance or from the class (microoptimization here)"
424
ret = getattr(self, att, self)
425
if ret is self: return getattr(self.__class__, att, default)
428
def hash_constraints(self):
429
"identify a task type for all the constraints relevant for the scheduler: precedence, file production"
431
sum = hash((self.__class__.__name__,
432
str(a('before', '')),
434
str(a('ext_in', '')),
435
str(a('ext_out', '')),
436
self.__class__.maxjobs))
439
def format_error(self):
440
"error message to display to the user (when a build fails)"
441
if getattr(self, "error_msg", None):
442
return self.error_msg
443
elif self.hasrun == CRASHED:
445
return " -> task failed (err #%d): %r" % (self.err_code, self)
446
except AttributeError:
447
return " -> task failed: %r" % self
448
elif self.hasrun == EXCEPTION:
450
elif self.hasrun == MISSING:
451
return " -> missing files: %r" % self
457
installation is performed by looking at the task attributes:
458
* install_path: installation path like "${PREFIX}/bin"
459
* filename: install the first node in the outputs as a file with a particular name, be certain to give os.sep
462
bld = self.generator.bld
463
d = self.attr('install')
465
if self.attr('install_path'):
466
lst = [a.relpath_gen(bld.srcnode) for a in self.outputs]
467
perm = self.attr('chmod', O644)
469
# if src is given, install the sources too
470
lst += [a.relpath_gen(bld.srcnode) for a in self.inputs]
471
if self.attr('filename'):
472
dir = self.install_path + self.attr('filename')
473
bld.install_as(self.install_path, lst[0], self.env, perm)
475
bld.install_files(self.install_path, lst, self.env, perm)
111
477
class Task(TaskBase):
112
"Task is the more common task. It has input nodes and output nodes"
113
def __init__(self, action_name, env, priority=5, normal=1):
114
TaskBase.__init__(self, priority, normal)
116
# name of the action associated to this task type
117
self.m_action = Action.g_actions[action_name]
478
"""The parent class is quite limited, in this version:
479
* file system interaction: input and output nodes
480
* persistence: do not re-execute tasks that have already run
481
* caching: same files can be saved and retrieved from a cache directory
483
implicit, like .c files depending on .h files
484
explicit, like the input nodes or the dep_nodes
485
environment variables, like the CXXFLAGS in self.env
488
def __init__(self, env, **kw):
489
TaskBase.__init__(self, **kw)
122
492
# inputs and outputs are nodes
123
493
# use setters when possible
128
self.m_scanner = Scan.g_default_scanner
130
# TODO get rid of this:
131
# default scanner parameter
132
global g_default_param
133
self.m_scanner_params = g_default_param
135
# additionally, you may define the following
136
# self.dep_vars = 'some_env_var'
500
# Additionally, you may define the following
501
#self.dep_vars = 'PREFIX DATADIR'
504
"string to display to the user"
506
src_str = ' '.join([a.nice_path(env) for a in self.inputs])
507
tgt_str = ' '.join([a.nice_path(env) for a in self.outputs])
508
if self.outputs: sep = ' -> '
510
return '%s: %s%s%s\n' % (self.__class__.__name__, src_str, sep, tgt_str)
513
return "".join(['\n\t{task: ', self.__class__.__name__, " ", ",".join([x.name for x in self.inputs]), " -> ", ",".join([x.name for x in self.outputs]), '}'])
516
"get a unique id: hash the node paths, the variant, the class, the function"
519
except AttributeError:
522
up(self.env.variant())
523
for x in self.inputs + self.outputs:
525
up(self.__class__.__name__)
526
up(Utils.h_fun(self.run))
527
self.uid = m.digest()
139
530
def set_inputs(self, inp):
140
if type(inp) is types.ListType: self.m_inputs = inp
141
else: self.m_inputs = [inp]
531
if type(inp) is types.ListType: self.inputs += inp
532
else: self.inputs.append(inp)
143
534
def set_outputs(self, out):
144
if type(out) is types.ListType: self.m_outputs = out
145
else: self.m_outputs = [out]
535
if type(out) is types.ListType: self.outputs += out
536
else: self.outputs.append(out)
147
538
def set_run_after(self, task):
148
"set (scheduler) dependency on another task"
539
"set (scheduler) order on another task"
149
540
# TODO: handle list or object
150
541
assert isinstance(task, TaskBase)
151
try: self.m_run_after.append(task)
152
except AttributeError: self.m_run_after = [task]
154
def get_run_after(self):
155
try: return self.m_run_after
156
except AttributeError: return []
542
self.run_after.append(task)
158
544
def add_file_dependency(self, filename):
159
545
"TODO user-provided file dependencies"
160
node = Params.g_build.m_current.find_build(filename)
161
try: self.m_deps_nodes.append(node)
162
except: self.m_deps_nodes = [node]
164
#------------ users are probably less interested in the following methods --------------#
546
node = self.generator.bld.current.find_resource(filename)
547
self.deps_nodes.append(node)
166
549
def signature(self):
167
# compute the result one time, and suppose the scanner.get_signature will give the good result
168
try: return self._sign_all
550
# compute the result one time, and suppose the scan_signature will give the good result
551
try: return self.cache_sig[0]
169
552
except AttributeError: pass
171
tree = Params.g_build
175
dep_sig = Params.sig_nil
178
scan = self.m_scanner
179
except AttributeError: # there is no scanner for the task
180
for x in self.m_inputs:
181
variant = x.variant(self.m_env)
182
v = tree.m_tstamp_variants[variant][x]
183
dep_sig = hash( (dep_sig, v) )
186
dep_sig = scan.get_signature(self)
190
try: act_sig = self.m_action.signature(self)
191
except AttributeError: act_sig = Object.sign_env_vars(self.m_env, self.m_action.m_vars)
196
var_sig = Object.sign_env_vars(self.m_env, self.dep_vars)
198
except AttributeError:
201
node_sig = Params.sig_nil
203
for x in self.dep_nodes:
204
variant = x.variant(self.m_env)
205
v = tree.m_tstamp_variants[variant][x]
206
node_sig = hash( (node_sig, v) )
208
except AttributeError:
211
# hash additional node dependencies
557
exp_sig = self.sig_explicit_deps()
561
imp_sig = self.scan and self.sig_implicit_deps() or SIG_NIL
565
var_sig = self.sig_vars()
568
# we now have the signature (first element) and the details (for debugging)
214
self.cache_sig = [ret, dep_sig, act_sig, var_sig, node_sig]
570
self.cache_sig = (ret, exp_sig, imp_sig, var_sig)
220
"wait for other tasks to complete"
221
if (not self.m_inputs) or (not self.m_outputs):
222
if not (not self.m_inputs) and (not self.m_outputs):
223
error("potentially grave error, task is invalid : no inputs or outputs")
226
# the scanner has its word to say
228
if not self.m_scanner.may_start(self):
230
except AttributeError:
233
# this is a dependency using the scheduler, as opposed to hash-based ones
234
for t in self.get_run_after():
240
"see if the task must be run or not"
573
def runnable_status(self):
574
"SKIP_ME RUN_ME or ASK_LATER"
241
575
#return 0 # benchmarking
243
tree = Params.g_build
246
# for tasks that have no inputs or outputs and are run all the time
247
if not self.m_inputs and not self.m_outputs:
248
self.m_dep_sig = Params.sig_nil
577
if self.inputs and (not self.outputs):
578
if not getattr(self.__class__, 'quiet', None):
579
warn("task is probably invalid (no inputs OR outputs): override in a Task subclass or set the attribute 'quiet' %r" % self)
581
for t in self.run_after:
586
bld = self.generator.bld
251
588
# look at the previous signature first
590
for node in self.outputs:
591
variant = node.variant(env)
593
time = bld.node_sigs[variant][node.id]
595
debug("task: task %r must run as the first node does not exist" % self)
599
# if one of the nodes does not exist, try to retrieve them from the cache
600
if time is None and self.outputs:
602
new_sig = self.signature()
604
debug("task: something is wrong, computing the task signature failed")
609
key = self.unique_id()
253
node = self.m_outputs[0]
254
variant = node.variant(self.m_env)
255
time = tree.m_tstamp_variants[variant][node]
256
key = hash( (variant, node, time, self.m_scanner.__class__.__name__) )
257
prev_sig = tree.m_sig_cache[key][0]
611
prev_sig = bld.task_sigs[key][0]
259
# an exception here means the object files do not exist
260
debug("task #%d should run as the first node does not exist" % self.m_idx, 'task')
262
# maybe we can just retrieve the object files from the cache then
263
ret = self.can_retrieve_cache(self.signature())
613
debug("task: task %r must run as it was never run before or the task code changed" % self)
616
#print "prev_sig is ", prev_sig
266
617
new_sig = self.signature()
268
619
# debug if asked to
270
self.debug_why(tree.m_sig_cache[key])
620
if Logs.verbose: self.debug_why(bld.task_sigs[key])
273
622
if new_sig != prev_sig:
274
# if the node has not changed, try to use the cache
275
ret = self.can_retrieve_cache(new_sig)
280
def update_stat(self):
281
"this is called after a sucessful task run"
282
tree = Params.g_build
627
"called after a successful task run"
628
bld = self.generator.bld
284
630
sig = self.signature()
287
for node in self.m_outputs:
633
for node in self.outputs:
288
634
variant = node.variant(env)
289
#if node in tree.m_tstamp_variants[variant]:
635
#if node in bld.node_sigs[variant]:
290
636
# print "variant is ", variant
291
# print "self sig is ", Params.vsig(tree.m_tstamp_variants[variant][node])
637
# print "self sig is ", Utils.view_sig(bld.node_sigs[variant][node])
293
639
# check if the node exists ..
295
os.stat(node.abspath(env))
297
error('a node was not produced for task %s %s' % (str(self.m_idx), node.abspath(env)))
640
os.stat(node.abspath(env))
300
642
# important, store the signature for the next run
301
tree.m_tstamp_variants[variant][node] = sig
643
bld.node_sigs[variant][node.id] = sig
303
645
# We could re-create the signature of the task with the signature of the outputs
304
646
# in practice, this means hashing the output files
305
647
# this is unnecessary
307
if Params.g_usecache:
648
if Options.cache_global:
308
649
ssig = sig.encode('hex')
309
dest = os.path.join(Params.g_usecache, ssig+'-'+str(cnt))
650
dest = os.path.join(Options.cache_global, ssig+'-'+str(cnt))
310
651
try: shutil.copy2(node.abspath(env), dest)
311
except IOError: warning('could not write the file to the cache')
652
except IOError: warn('Could not write the file to the cache')
314
# keep the signatures in the first node
315
node = self.m_outputs[0]
316
variant = node.variant(self.m_env)
317
time = tree.m_tstamp_variants[variant][node]
318
key = hash( (variant, node, time, self.m_scanner.__class__.__name__) )
320
tree.set_sig_cache(key, val)
324
def can_retrieve_cache(self, sig):
655
bld.task_sigs[self.unique_id()] = self.cache_sig
658
def can_retrieve_cache(self):
325
659
"""Retrieve build nodes from the cache - the file time stamps are updated
326
for cleaning the least used files from the cache dir - be careful when overriding"""
327
if not Params.g_usecache: return None
328
if Params.g_options.nocache: return None
660
for cleaning the least used files from the cache dir - be careful when overridding"""
661
if not Options.cache_global: return None
662
if Options.options.nocache: return None
331
665
sig = self.signature()
335
for node in self.m_outputs:
336
variant = node.variant(env)
668
for node in self.outputs:
669
variant = node.variant(env)
338
ssig = sig.encode('hex')
339
orig = os.path.join(Params.g_usecache, ssig+'-'+str(cnt))
671
ssig = sig.encode('hex')
672
orig = os.path.join(Options.cache_global, ssig+'-'+str(cnt))
340
674
shutil.copy2(orig, node.abspath(env))
343
# what i would like to do is to limit the max size of the cache, using either
344
# df (file system full) or a fixed size (like say no more than 400Mb of cache)
345
# removing the files would be done by order of timestamps (TODO ITA)
675
# mark the cache file as used recently (modified)
346
676
os.utime(orig, None)
677
except (OSError, IOError):
678
debug('task: failed retrieving file')
349
Params.g_build.m_tstamp_variants[variant][node] = sig
350
if not Runner.g_quiet: Params.pprint('GREEN', 'restored from cache %s' % node.bldpath(env))
352
debug("failed retrieving file", 'task')
682
self.generator.bld.node_sigs[variant][node.id] = sig
683
self.generator.bld.printout('restoring from cache %r\n' % node.bldpath(env))
357
try: self.m_action.prepare(self)
358
except AttributeError: pass
361
return self.m_action.run(self)
363
def get_display(self):
364
if self.m_display: return self.m_display
365
self.m_display=self.m_action.get_str(self)
366
return self.m_display
369
return self.m_action.m_color
371
def debug_info(self):
373
ret.append('-- task details begin --')
374
ret.append('action: %s' % str(self.m_action))
375
ret.append('idx: %s' % str(self.m_idx))
376
ret.append('source: %s' % str(self.m_inputs))
377
ret.append('target: %s' % str(self.m_outputs))
378
ret.append('-- task details end --')
379
return '\n'.join(ret)
381
def debug(self, level=0):
383
if level>0: fun=Params.error
384
fun(self.debug_info())
386
686
def debug_why(self, old_sigs):
387
687
"explains why a task is run"
389
689
new_sigs = self.cache_sig
392
debug("Task %s must run: %s" % (self.m_idx, old_sigs[0] != new_sigs[0]), 'task')
393
if (new_sigs[1] != old_sigs[1]):
394
debug(' -> A source file (or a dependency) has changed %s %s' % (v(old_sigs[1]), v(new_sigs[1])), 'task')
395
if (new_sigs[2] != old_sigs[2]):
396
debug(' -> An environment variable has changed %s %s' % (v(old_sigs[2]), v(new_sigs[2])), 'task')
397
if (new_sigs[3] != old_sigs[3]):
398
debug(' -> A manual dependency has changed %s %s' % (v(old_sigs[3]), v(new_sigs[3])), 'task')
399
if (new_sigs[4] != old_sigs[4]):
400
debug(' -> A user-given environment variable has changed %s %s' % (v(old_sigs[4]), v(new_sigs[4])), 'task')
402
class TaskCmd(TaskBase):
403
"TaskCmd executes commands. Instances always execute their function."
404
def __init__(self, fun, env, priority):
405
TaskBase.__init__(self, priority)
409
self.display = "* executing: "+self.fun.__name__
410
def debug_info(self):
411
return 'TaskCmd:fun %s' % self.fun.__name__
413
return 'TaskCmd:fun %s' % self.fun.__name__
691
return x.encode('hex')
693
debug("Task %r" % self)
694
msgs = ['Task must run', '* Source file or manual dependency', '* Implicit dependency', '* Environment variable']
695
tmp = 'task: -> %s: %s %s'
696
for x in xrange(len(msgs)):
697
if (new_sigs[x] != old_sigs[x]):
698
debug(tmp % (msgs[x], v(old_sigs[x]), v(new_sigs[x])))
700
def sig_explicit_deps(self):
701
bld = self.generator.bld
705
for x in self.inputs:
706
variant = x.variant(self.env)
707
m.update(bld.node_sigs[variant][x.id])
709
# additional nodes to depend on, if provided
710
for x in getattr(self, 'dep_nodes', []):
711
variant = x.variant(self.env)
712
v = bld.node_sigs[variant][x.id]
715
# manual dependencies, they can slow down the builds
717
additional_deps = bld.deps_man
718
except AttributeError:
721
for x in self.inputs + self.outputs:
723
d = additional_deps[x.id]
727
d = d() # dependency is a function, call it
730
if isinstance(v, Node.Node):
732
variant = v.variant(self.env)
734
v = bld.node_sigs[variant][v.id]
735
except KeyError: # make it fatal?
742
bld = self.generator.bld
745
# dependencies on the environment vars
746
act_sig = bld.hash_env_vars(env, self.__class__.vars)
749
# additional variable dependencies, if provided
750
dep_vars = getattr(self, 'dep_vars', None)
752
m.update(bld.hash_env_vars(env, dep_vars))
756
#def scan(self, node):
757
# """this method returns a tuple containing:
758
# * a list of nodes corresponding to real files
759
# * a list of names for files not found in path_lst
760
# the input parameters may have more parameters that the ones used below
765
# compute the signature, recompute it if there is no match in the cache
766
def sig_implicit_deps(self):
767
"the signature obtained may not be the one if the files have changed, we do it in two steps"
769
bld = self.generator.bld
771
# get the task signatures from previous runs
772
key = self.unique_id()
773
prev_sigs = bld.task_sigs.get(key, ())
776
if prev_sigs[2] == self.compute_sig_implicit_deps():
781
# no previous run or the signature of the dependencies has changed, rescan the dependencies
782
(nodes, names) = self.scan()
784
debug('deps: scanner for %s returned %s %s' % (str(self), str(nodes), str(names)))
786
# store the dependencies in the cache
787
bld.node_deps[key] = nodes
788
bld.raw_deps[key] = names
790
# recompute the signature and return it
791
sig = self.compute_sig_implicit_deps()
795
def compute_sig_implicit_deps(self):
796
"""it is intented for .cpp and inferred .h files
797
there is a single list (no tree traversal)
798
this is the hot spot so ... do not touch"""
802
bld = self.generator.bld
803
tstamp = bld.node_sigs
806
for k in bld.node_deps.get(self.unique_id(), []) + self.inputs:
807
# unlikely but necessary if it happens
808
if not k.parent.id in bld.cache_scanned_folders:
811
if k.id & 3 == Node.FILE:
814
upd(tstamp[env.variant()][k.id])
823
reg_act = re.compile(r"(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})", re.M)
824
def compile_fun(name, line):
825
"""Compiles a string (once) into a function, eg:
826
simple_task_type('c++', '${CXX} -o ${TGT[0]} ${SRC} -I ${SRC[0].parent.bldpath()}')
828
The env variables (CXX, ..) on the task must not hold dicts (order)
829
The reserved keywords TGT and SRC represent the task input and output nodes
834
if g('dollar'): return "$"
835
elif g('subst'): extr.append((g('var'), g('code'))); return "%s"
838
line = reg_act.sub(repl, line)
843
for (var, meth) in extr:
845
if meth: app('task.inputs%s' % meth)
846
else: app('" ".join([a.srcpath(env) for a in task.inputs])')
848
if meth: app('task.outputs%s' % meth)
849
else: app('" ".join([a.bldpath(env) for a in task.outputs])')
851
if not var in dvars: dvars.append(var)
853
if parm: parm = "%% (%s) " % (',\n\t\t'.join(parm))
859
wd = getattr(task, 'cwd', None)
862
return task.generator.bld.exec_command(cmd, cwd=wd)
865
debug('action: %s' % c)
866
return (funex(c), dvars)
868
def simple_task_type(name, line, color='GREEN', vars=[], ext_in=[], ext_out=[], before=[], after=[]):
869
"""return a new Task subclass with the function run compiled from the line given"""
870
(fun, dvars) = compile_fun(name, line)
872
return task_type_from_func(name, fun, vars or dvars, color, ext_in, ext_out, before, after)
874
def task_type_from_func(name, func, vars=[], color='GREEN', ext_in=[], ext_out=[], before=[], after=[]):
875
"""return a new Task subclass with the function run compiled from the line given"""
881
'ext_in': Utils.to_list(ext_in),
882
'ext_out': Utils.to_list(ext_out),
883
'before': Utils.to_list(before),
884
'after': Utils.to_list(after),
887
cls = type(Task)(name, (Task,), params)
888
TaskBase.classes[name] = cls
892
"""Set all task instances of this class to be executed whenever a build is started
893
The task signature is calculated, but the result of the comparation between
894
task signatures is bypassed
896
old = cls.runnable_status
900
cls.runnable_status = always
902
def update_outputs(cls):
903
"""When a command is always run, it is possible that the output only change
904
sometimes. By default the build node have as a hash the signature of the task
905
which may not change. With this, the output nodes (produced) are hashed,
906
and the hashes are set to the build nodes
908
This may avoid unnecessary recompilations, but it uses more resources
909
(hashing the output files) so it is not used by default
911
old_post_run = cls.post_run
914
bld = self.outputs[0].__class__.bld
915
bld.node_sigs[self.env.variant()][self.outputs[0].id] = \
916
Utils.h_file(self.outputs[0].abspath(self.env))
917
cls.post_run = post_run
added
removed
wafadmin/Task.py
