« back to all changes in this revision
Viewing changes to waflib/Task.py
- browse files at revision 1.1.10
- compare with another revision
- download diff
- download tarball
- view history from revision 1.1.10
- Committer: Package Import Robot
- Author(s): Reinhard Tartler
- Date: 2012-11-25 19:23:15 UTC
- mto: This revision was merged to the branch mainline in revision 51.
- Revision ID: package-import@ubuntu.com-20121125192315-m9z6nu9wwlzrrz9z
Import upstream version 0.8+dfsg
- files added:
- waflib
- waflib/Tools
- waflib/extras
- files modified:
- waf
added
removed
waflib/Task.py
1
#! /usr/bin/env python
2
# encoding: utf-8
3
# WARNING! Do not edit! http://waf.googlecode.com/svn/docs/wafbook/single.html#_obtaining_the_waf_file
4
5
import sys
6
if sys.hexversion < 0x020400f0: from sets import Set as set
7
import os,shutil,re,tempfile
8
from waflib import Utils,Logs,Errors
9
NOT_RUN=0
10
MISSING=1
11
CRASHED=2
12
EXCEPTION=3
13
SKIPPED=8
14
SUCCESS=9
15
ASK_LATER=-1
16
SKIP_ME=-2
17
RUN_ME=-3
18
COMPILE_TEMPLATE_SHELL='''
19
def f(tsk):
20
env = tsk.env
21
gen = tsk.generator
22
bld = gen.bld
23
wd = getattr(tsk, 'cwd', None)
24
p = env.get_flat
25
tsk.last_cmd = cmd = \'\'\' %s \'\'\' % s
26
return tsk.exec_command(cmd, cwd=wd, env=env.env or None)
27
'''
28
COMPILE_TEMPLATE_NOSHELL='''
29
def f(tsk):
30
env = tsk.env
31
gen = tsk.generator
32
bld = gen.bld
33
wd = getattr(tsk, 'cwd', None)
34
def to_list(xx):
35
if isinstance(xx, str): return [xx]
36
return xx
37
tsk.last_cmd = lst = []
38
%s
39
lst = [x for x in lst if x]
40
return tsk.exec_command(lst, cwd=wd, env=env.env or None)
41
'''
42
def cache_outputs(cls):
43
m1=cls.run
44
def run(self):
45
bld=self.generator.bld
46
if bld.cache_global and not bld.nocache:
47
if self.can_retrieve_cache():
48
return 0
49
return m1(self)
50
cls.run=run
51
m2=cls.post_run
52
def post_run(self):
53
bld=self.generator.bld
54
ret=m2(self)
55
if bld.cache_global and not bld.nocache:
56
self.put_files_cache()
57
return ret
58
cls.post_run=post_run
59
return cls
60
classes={}
61
class store_task_type(type):
62
def __init__(cls,name,bases,dict):
63
super(store_task_type,cls).__init__(name,bases,dict)
64
name=cls.__name__
65
if name.endswith('_task'):
66
name=name.replace('_task','')
67
if name!='evil'and name!='TaskBase':
68
global classes
69
if getattr(cls,'run_str',None):
70
(f,dvars)=compile_fun(cls.run_str,cls.shell)
71
cls.hcode=cls.run_str
72
cls.run_str=None
73
cls.run=f
74
cls.vars=list(set(cls.vars+dvars))
75
cls.vars.sort()
76
elif getattr(cls,'run',None)and not'hcode'in cls.__dict__:
77
cls.hcode=Utils.h_fun(cls.run)
78
if not getattr(cls,'nocache',None):
79
cls=cache_outputs(cls)
80
classes[name]=cls
81
evil=store_task_type('evil',(object,),{})
82
class TaskBase(evil):
83
color='GREEN'
84
ext_in=[]
85
ext_out=[]
86
before=[]
87
after=[]
88
hcode=''
89
def __init__(self,*k,**kw):
90
self.hasrun=NOT_RUN
91
try:
92
self.generator=kw['generator']
93
except KeyError:
94
self.generator=self
95
def __repr__(self):
96
return'\n\t{task %r: %s %s}'%(self.__class__.__name__,id(self),str(getattr(self,'fun','')))
97
def __str__(self):
98
if hasattr(self,'fun'):
99
return'executing: %s\n'%self.fun.__name__
100
return self.__class__.__name__+'\n'
101
def __hash__(self):
102
return id(self)
103
def exec_command(self,cmd,**kw):
104
bld=self.generator.bld
105
try:
106
if not kw.get('cwd',None):
107
kw['cwd']=bld.cwd
108
except AttributeError:
109
bld.cwd=kw['cwd']=bld.variant_dir
110
return bld.exec_command(cmd,**kw)
111
def runnable_status(self):
112
return RUN_ME
113
def process(self):
114
m=self.master
115
if m.stop:
116
m.out.put(self)
117
return
118
try:
119
del self.generator.bld.task_sigs[self.uid()]
120
except:
121
pass
122
try:
123
self.generator.bld.returned_tasks.append(self)
124
self.log_display(self.generator.bld)
125
ret=self.run()
126
except Exception:
127
self.err_msg=Utils.ex_stack()
128
self.hasrun=EXCEPTION
129
m.error_handler(self)
130
m.out.put(self)
131
return
132
if ret:
133
self.err_code=ret
134
self.hasrun=CRASHED
135
else:
136
try:
137
self.post_run()
138
except Errors.WafError:
139
pass
140
except Exception:
141
self.err_msg=Utils.ex_stack()
142
self.hasrun=EXCEPTION
143
else:
144
self.hasrun=SUCCESS
145
if self.hasrun!=SUCCESS:
146
m.error_handler(self)
147
m.out.put(self)
148
def run(self):
149
if hasattr(self,'fun'):
150
return self.fun(self)
151
return 0
152
def post_run(self):
153
pass
154
def log_display(self,bld):
155
bld.to_log(self.display())
156
def display(self):
157
col1=Logs.colors(self.color)
158
col2=Logs.colors.NORMAL
159
master=self.master
160
def cur():
161
tmp=-1
162
if hasattr(master,'ready'):
163
tmp-=master.ready.qsize()
164
return master.processed+tmp
165
if self.generator.bld.progress_bar==1:
166
return self.generator.bld.progress_line(cur(),master.total,col1,col2)
167
if self.generator.bld.progress_bar==2:
168
ela=str(self.generator.bld.timer)
169
try:
170
ins=','.join([n.name for n in self.inputs])
171
except AttributeError:
172
ins=''
173
try:
174
outs=','.join([n.name for n in self.outputs])
175
except AttributeError:
176
outs=''
177
return'|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n'%(master.total,cur(),ins,outs,ela)
178
s=str(self)
179
if not s:
180
return None
181
total=master.total
182
n=len(str(total))
183
fs='[%%%dd/%%%dd] %%s%%s%%s'%(n,n)
184
return fs%(cur(),total,col1,s,col2)
185
def attr(self,att,default=None):
186
ret=getattr(self,att,self)
187
if ret is self:return getattr(self.__class__,att,default)
188
return ret
189
def hash_constraints(self):
190
cls=self.__class__
191
tup=(str(cls.before),str(cls.after),str(cls.ext_in),str(cls.ext_out),cls.__name__,cls.hcode)
192
h=hash(tup)
193
return h
194
def format_error(self):
195
msg=getattr(self,'last_cmd','')
196
if getattr(self,"err_msg",None):
197
return self.err_msg
198
elif self.hasrun==CRASHED:
199
try:
200
return' -> task failed (exit status %r): %r\n%r'%(self.err_code,self,msg)
201
except AttributeError:
202
return' -> task failed: %r\n%r'%(self,msg)
203
elif self.hasrun==MISSING:
204
return' -> missing files: %r\n%r'%(self,msg)
205
else:
206
return'?'
207
def colon(self,var1,var2):
208
tmp=self.env[var1]
209
if isinstance(var2,str):
210
it=self.env[var2]
211
else:
212
it=var2
213
if isinstance(tmp,str):
214
return[tmp%x for x in it]
215
else:
216
if Logs.verbose and not tmp and it:
217
Logs.warn('Missing env variable %r for task %r (generator %r)'%(var1,self,self.generator))
218
lst=[]
219
for y in it:
220
lst.extend(tmp)
221
lst.append(y)
222
return lst
223
class Task(TaskBase):
224
vars=[]
225
shell=False
226
def __init__(self,*k,**kw):
227
TaskBase.__init__(self,*k,**kw)
228
self.env=kw['env']
229
self.inputs=[]
230
self.outputs=[]
231
self.dep_nodes=[]
232
self.run_after=set([])
233
def __str__(self):
234
env=self.env
235
src_str=' '.join([a.nice_path(env)for a in self.inputs])
236
tgt_str=' '.join([a.nice_path(env)for a in self.outputs])
237
if self.outputs:sep=' -> '
238
else:sep=''
239
return'%s: %s%s%s\n'%(self.__class__.__name__.replace('_task',''),src_str,sep,tgt_str)
240
def __repr__(self):
241
return"".join(['\n\t{task %r: '%id(self),self.__class__.__name__," ",",".join([x.name for x in self.inputs])," -> ",",".join([x.name for x in self.outputs]),'}'])
242
def uid(self):
243
try:
244
return self.uid_
245
except AttributeError:
246
m=Utils.md5()
247
up=m.update
248
up(self.__class__.__name__)
249
for x in self.inputs+self.outputs:
250
up(x.abspath())
251
self.uid_=m.digest()
252
return self.uid_
253
def set_inputs(self,inp):
254
if isinstance(inp,list):self.inputs+=inp
255
else:self.inputs.append(inp)
256
def set_outputs(self,out):
257
if isinstance(out,list):self.outputs+=out
258
else:self.outputs.append(out)
259
def set_run_after(self,task):
260
assert isinstance(task,TaskBase)
261
self.run_after.add(task)
262
def signature(self):
263
try:return self.cache_sig
264
except AttributeError:pass
265
self.m=Utils.md5()
266
self.m.update(self.hcode)
267
self.sig_explicit_deps()
268
self.sig_vars()
269
if self.scan:
270
try:
271
self.sig_implicit_deps()
272
except Errors.TaskRescan:
273
return self.signature()
274
ret=self.cache_sig=self.m.digest()
275
return ret
276
def runnable_status(self):
277
for t in self.run_after:
278
if not t.hasrun:
279
return ASK_LATER
280
bld=self.generator.bld
281
try:
282
new_sig=self.signature()
283
except Errors.TaskNotReady:
284
return ASK_LATER
285
key=self.uid()
286
try:
287
prev_sig=bld.task_sigs[key]
288
except KeyError:
289
Logs.debug("task: task %r must run as it was never run before or the task code changed"%self)
290
return RUN_ME
291
for node in self.outputs:
292
try:
293
if node.sig!=new_sig:
294
return RUN_ME
295
except AttributeError:
296
Logs.debug("task: task %r must run as the output nodes do not exist"%self)
297
return RUN_ME
298
if new_sig!=prev_sig:
299
return RUN_ME
300
return SKIP_ME
301
def post_run(self):
302
bld=self.generator.bld
303
sig=self.signature()
304
for node in self.outputs:
305
try:
306
os.stat(node.abspath())
307
except OSError:
308
self.hasrun=MISSING
309
self.err_msg='-> missing file: %r'%node.abspath()
310
raise Errors.WafError(self.err_msg)
311
node.sig=sig
312
bld.task_sigs[self.uid()]=self.cache_sig
313
def sig_explicit_deps(self):
314
bld=self.generator.bld
315
upd=self.m.update
316
for x in self.inputs+self.dep_nodes:
317
try:
318
upd(x.get_bld_sig())
319
except(AttributeError,TypeError):
320
raise Errors.WafError('Missing node signature for %r (required by %r)'%(x,self))
321
if bld.deps_man:
322
additional_deps=bld.deps_man
323
for x in self.inputs+self.outputs:
324
try:
325
d=additional_deps[id(x)]
326
except KeyError:
327
continue
328
for v in d:
329
if isinstance(v,bld.root.__class__):
330
try:
331
v=v.get_bld_sig()
332
except AttributeError:
333
raise Errors.WafError('Missing node signature for %r (required by %r)'%(v,self))
334
elif hasattr(v,'__call__'):
335
v=v()
336
upd(v)
337
return self.m.digest()
338
def sig_vars(self):
339
bld=self.generator.bld
340
env=self.env
341
upd=self.m.update
342
act_sig=bld.hash_env_vars(env,self.__class__.vars)
343
upd(act_sig)
344
dep_vars=getattr(self,'dep_vars',None)
345
if dep_vars:
346
upd(bld.hash_env_vars(env,dep_vars))
347
return self.m.digest()
348
scan=None
349
def sig_implicit_deps(self):
350
bld=self.generator.bld
351
key=self.uid()
352
prev=bld.task_sigs.get((key,'imp'),[])
353
if prev:
354
try:
355
if prev==self.compute_sig_implicit_deps():
356
return prev
357
except:
358
for x in bld.node_deps.get(self.uid(),[]):
359
if x.is_child_of(bld.srcnode):
360
try:
361
os.stat(x.abspath())
362
except:
363
try:
364
del x.parent.children[x.name]
365
except:
366
pass
367
del bld.task_sigs[(key,'imp')]
368
raise Errors.TaskRescan('rescan')
369
(nodes,names)=self.scan()
370
if Logs.verbose:
371
Logs.debug('deps: scanner for %s returned %s %s'%(str(self),str(nodes),str(names)))
372
bld.node_deps[key]=nodes
373
bld.raw_deps[key]=names
374
self.are_implicit_nodes_ready()
375
try:
376
bld.task_sigs[(key,'imp')]=sig=self.compute_sig_implicit_deps()
377
except:
378
if Logs.verbose:
379
for k in bld.node_deps.get(self.uid(),[]):
380
try:
381
k.get_bld_sig()
382
except:
383
Logs.warn('Missing signature for node %r (may cause rebuilds)'%k)
384
else:
385
return sig
386
def compute_sig_implicit_deps(self):
387
upd=self.m.update
388
bld=self.generator.bld
389
self.are_implicit_nodes_ready()
390
for k in bld.node_deps.get(self.uid(),[]):
391
upd(k.get_bld_sig())
392
return self.m.digest()
393
def are_implicit_nodes_ready(self):
394
bld=self.generator.bld
395
try:
396
cache=bld.dct_implicit_nodes
397
except:
398
bld.dct_implicit_nodes=cache={}
399
try:
400
dct=cache[bld.cur]
401
except KeyError:
402
dct=cache[bld.cur]={}
403
for tsk in bld.cur_tasks:
404
for x in tsk.outputs:
405
dct[x]=tsk
406
modified=False
407
for x in bld.node_deps.get(self.uid(),[]):
408
if x in dct:
409
self.run_after.add(dct[x])
410
modified=True
411
if modified:
412
for tsk in self.run_after:
413
if not tsk.hasrun:
414
raise Errors.TaskNotReady('not ready')
415
def can_retrieve_cache(self):
416
if not getattr(self,'outputs',None):
417
return None
418
sig=self.signature()
419
ssig=Utils.to_hex(self.uid())+Utils.to_hex(sig)
420
dname=os.path.join(self.generator.bld.cache_global,ssig)
421
try:
422
t1=os.stat(dname).st_mtime
423
except OSError:
424
return None
425
for node in self.outputs:
426
orig=os.path.join(dname,node.name)
427
try:
428
shutil.copy2(orig,node.abspath())
429
os.utime(orig,None)
430
except(OSError,IOError):
431
Logs.debug('task: failed retrieving file')
432
return None
433
try:
434
t2=os.stat(dname).st_mtime
435
except OSError:
436
return None
437
if t1!=t2:
438
return None
439
for node in self.outputs:
440
node.sig=sig
441
if self.generator.bld.progress_bar<1:
442
self.generator.bld.to_log('restoring from cache %r\n'%node.abspath())
443
self.cached=True
444
return True
445
def put_files_cache(self):
446
if getattr(self,'cached',None):
447
return None
448
if not getattr(self,'outputs',None):
449
return None
450
sig=self.signature()
451
ssig=Utils.to_hex(self.uid())+Utils.to_hex(sig)
452
dname=os.path.join(self.generator.bld.cache_global,ssig)
453
tmpdir=tempfile.mkdtemp(prefix=self.generator.bld.cache_global+os.sep+'waf')
454
try:
455
shutil.rmtree(dname)
456
except:
457
pass
458
try:
459
for node in self.outputs:
460
dest=os.path.join(tmpdir,node.name)
461
shutil.copy2(node.abspath(),dest)
462
except(OSError,IOError):
463
try:
464
shutil.rmtree(tmpdir)
465
except:
466
pass
467
else:
468
try:
469
os.rename(tmpdir,dname)
470
except OSError:
471
try:
472
shutil.rmtree(tmpdir)
473
except:
474
pass
475
else:
476
try:
477
os.chmod(dname,Utils.O755)
478
except:
479
pass
480
def is_before(t1,t2):
481
to_list=Utils.to_list
482
for k in to_list(t2.ext_in):
483
if k in to_list(t1.ext_out):
484
return 1
485
if t1.__class__.__name__ in to_list(t2.after):
486
return 1
487
if t2.__class__.__name__ in to_list(t1.before):
488
return 1
489
return 0
490
def set_file_constraints(tasks):
491
ins=Utils.defaultdict(set)
492
outs=Utils.defaultdict(set)
493
for x in tasks:
494
for a in getattr(x,'inputs',[])+getattr(x,'dep_nodes',[]):
495
ins[id(a)].add(x)
496
for a in getattr(x,'outputs',[]):
497
outs[id(a)].add(x)
498
links=set(ins.keys()).intersection(outs.keys())
499
for k in links:
500
for a in ins[k]:
501
a.run_after.update(outs[k])
502
def set_precedence_constraints(tasks):
503
cstr_groups=Utils.defaultdict(list)
504
for x in tasks:
505
h=x.hash_constraints()
506
cstr_groups[h].append(x)
507
keys=list(cstr_groups.keys())
508
maxi=len(keys)
509
for i in range(maxi):
510
t1=cstr_groups[keys[i]][0]
511
for j in range(i+1,maxi):
512
t2=cstr_groups[keys[j]][0]
513
if is_before(t1,t2):
514
a=i
515
b=j
516
elif is_before(t2,t1):
517
a=j
518
b=i
519
else:
520
continue
521
for x in cstr_groups[keys[b]]:
522
x.run_after.update(cstr_groups[keys[a]])
523
def funex(c):
524
dc={}
525
exec(c,dc)
526
return dc['f']
527
reg_act=re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})",re.M)
528
def compile_fun_shell(line):
529
extr=[]
530
def repl(match):
531
g=match.group
532
if g('dollar'):return"$"
533
elif g('backslash'):return'\\\\'
534
elif g('subst'):extr.append((g('var'),g('code')));return"%s"
535
return None
536
line=reg_act.sub(repl,line)or line
537
parm=[]
538
dvars=[]
539
app=parm.append
540
for(var,meth)in extr:
541
if var=='SRC':
542
if meth:app('tsk.inputs%s'%meth)
543
else:app('" ".join([a.path_from(bld.bldnode) for a in tsk.inputs])')
544
elif var=='TGT':
545
if meth:app('tsk.outputs%s'%meth)
546
else:app('" ".join([a.path_from(bld.bldnode) for a in tsk.outputs])')
547
elif meth:
548
if meth.startswith(':'):
549
m=meth[1:]
550
if m=='SRC':
551
m='[a.path_from(bld.bldnode) for a in tsk.inputs]'
552
elif m=='TGT':
553
m='[a.path_from(bld.bldnode) for a in tsk.outputs]'
554
elif m[:3]not in('tsk','gen','bld'):
555
dvars.extend([var,meth[1:]])
556
m='%r'%m
557
app('" ".join(tsk.colon(%r, %s))'%(var,m))
558
else:
559
app('%s%s'%(var,meth))
560
else:
561
if not var in dvars:dvars.append(var)
562
app("p('%s')"%var)
563
if parm:parm="%% (%s) "%(',\n\t\t'.join(parm))
564
else:parm=''
565
c=COMPILE_TEMPLATE_SHELL%(line,parm)
566
Logs.debug('action: %s'%c)
567
return(funex(c),dvars)
568
def compile_fun_noshell(line):
569
extr=[]
570
def repl(match):
571
g=match.group
572
if g('dollar'):return"$"
573
elif g('subst'):extr.append((g('var'),g('code')));return"<<|@|>>"
574
return None
575
line2=reg_act.sub(repl,line)
576
params=line2.split('<<|@|>>')
577
assert(extr)
578
buf=[]
579
dvars=[]
580
app=buf.append
581
for x in range(len(extr)):
582
params[x]=params[x].strip()
583
if params[x]:
584
app("lst.extend(%r)"%params[x].split())
585
(var,meth)=extr[x]
586
if var=='SRC':
587
if meth:app('lst.append(tsk.inputs%s)'%meth)
588
else:app("lst.extend([a.path_from(bld.bldnode) for a in tsk.inputs])")
589
elif var=='TGT':
590
if meth:app('lst.append(tsk.outputs%s)'%meth)
591
else:app("lst.extend([a.path_from(bld.bldnode) for a in tsk.outputs])")
592
elif meth:
593
if meth.startswith(':'):
594
m=meth[1:]
595
if m=='SRC':
596
m='[a.path_from(bld.bldnode) for a in tsk.inputs]'
597
elif m=='TGT':
598
m='[a.path_from(bld.bldnode) for a in tsk.outputs]'
599
elif m[:3]not in('tsk','gen','bld'):
600
dvars.extend([var,m])
601
m='%r'%m
602
app('lst.extend(tsk.colon(%r, %s))'%(var,m))
603
else:
604
app('lst.extend(gen.to_list(%s%s))'%(var,meth))
605
else:
606
app('lst.extend(to_list(env[%r]))'%var)
607
if not var in dvars:dvars.append(var)
608
if extr:
609
if params[-1]:
610
app("lst.extend(%r)"%params[-1].split())
611
fun=COMPILE_TEMPLATE_NOSHELL%"\n\t".join(buf)
612
Logs.debug('action: %s'%fun)
613
return(funex(fun),dvars)
614
def compile_fun(line,shell=False):
615
if line.find('<')>0 or line.find('>')>0 or line.find('&&')>0:
616
shell=True
617
if shell:
618
return compile_fun_shell(line)
619
else:
620
return compile_fun_noshell(line)
621
def task_factory(name,func=None,vars=None,color='GREEN',ext_in=[],ext_out=[],before=[],after=[],shell=False,scan=None):
622
params={'vars':vars or[],'color':color,'name':name,'ext_in':Utils.to_list(ext_in),'ext_out':Utils.to_list(ext_out),'before':Utils.to_list(before),'after':Utils.to_list(after),'shell':shell,'scan':scan,}
623
if isinstance(func,str):
624
params['run_str']=func
625
else:
626
params['run']=func
627
cls=type(Task)(name,(Task,),params)
628
global classes
629
classes[name]=cls
630
return cls
631
def always_run(cls):
632
old=cls.runnable_status
633
def always(self):
634
ret=old(self)
635
if ret==SKIP_ME:
636
ret=RUN_ME
637
return ret
638
cls.runnable_status=always
639
return cls
640
def update_outputs(cls):
641
old_post_run=cls.post_run
642
def post_run(self):
643
old_post_run(self)
644
for node in self.outputs:
645
node.sig=Utils.h_file(node.abspath())
646
self.generator.bld.task_sigs[node.abspath()]=self.uid()
647
cls.post_run=post_run
648
old_runnable_status=cls.runnable_status
649
def runnable_status(self):
650
status=old_runnable_status(self)
651
if status!=RUN_ME:
652
return status
653
try:
654
bld=self.generator.bld
655
prev_sig=bld.task_sigs[self.uid()]
656
if prev_sig==self.signature():
657
for x in self.outputs:
658
if not x.sig or bld.task_sigs[x.abspath()]!=self.uid():
659
return RUN_ME
660
return SKIP_ME
661
except KeyError:
662
pass
663
except IndexError:
664
pass
665
except AttributeError:
666
pass
667
return RUN_ME
668
cls.runnable_status=runnable_status
669
return cls
Loggerhead is a web-based interface for Breezy
Version: 2.0.1