« back to all changes in this revision
Viewing changes to waflib/Tools/c_preproc.py
- browse files at revision 1.1.10
- compare with another revision
- download diff
- download tarball
- view history from revision 1.1.10
- Committer: Package Import Robot
- Author(s): Reinhard Tartler
- Date: 2012-11-25 19:23:15 UTC
- mto: This revision was merged to the branch mainline in revision 51.
- Revision ID: package-import@ubuntu.com-20121125192315-m9z6nu9wwlzrrz9z
Import upstream version 0.8+dfsg
- files added:
- waflib
- waflib/Tools
- waflib/extras
- files modified:
- waf
added
removed
waflib/Tools/c_preproc.py
1
#! /usr/bin/env python
2
# encoding: utf-8
3
# WARNING! Do not edit! http://waf.googlecode.com/svn/docs/wafbook/single.html#_obtaining_the_waf_file
4
5
import sys
6
if sys.hexversion < 0x020400f0: from sets import Set as set
7
import re,sys,os,string,traceback
8
from waflib import Logs,Build,Utils,Errors
9
from waflib.Logs import debug,error
10
class PreprocError(Errors.WafError):
11
pass
12
POPFILE='-'
13
recursion_limit=150
14
go_absolute=False
15
standard_includes=['/usr/include']
16
if Utils.is_win32:
17
standard_includes=[]
18
use_trigraphs=0
19
strict_quotes=0
20
g_optrans={'not':'!','and':'&&','bitand':'&','and_eq':'&=','or':'||','bitor':'|','or_eq':'|=','xor':'^','xor_eq':'^=','compl':'~',}
21
re_lines=re.compile('^[ \t]*(#|%:)[ \t]*(ifdef|ifndef|if|else|elif|endif|include|import|define|undef|pragma)[ \t]*(.*)\r*$',re.IGNORECASE|re.MULTILINE)
22
re_mac=re.compile("^[a-zA-Z_]\w*")
23
re_fun=re.compile('^[a-zA-Z_][a-zA-Z0-9_]*[(]')
24
re_pragma_once=re.compile('^\s*once\s*',re.IGNORECASE)
25
re_nl=re.compile('\\\\\r*\n',re.MULTILINE)
26
re_cpp=re.compile(r"""(/\*[^*]*\*+([^/*][^*]*\*+)*/)|//[^\n]*|("(\\.|[^"\\])*"|'(\\.|[^'\\])*'|.[^/"'\\]*)""",re.MULTILINE)
27
trig_def=[('??'+a,b)for a,b in zip("=-/!'()<>",r'#~\|^[]{}')]
28
chr_esc={'0':0,'a':7,'b':8,'t':9,'n':10,'f':11,'v':12,'r':13,'\\':92,"'":39}
29
NUM='i'
30
OP='O'
31
IDENT='T'
32
STR='s'
33
CHAR='c'
34
tok_types=[NUM,STR,IDENT,OP]
35
exp_types=[r"""0[xX](?P<hex>[a-fA-F0-9]+)(?P<qual1>[uUlL]*)|L*?'(?P<char>(\\.|[^\\'])+)'|(?P<n1>\d+)[Ee](?P<exp0>[+-]*?\d+)(?P<float0>[fFlL]*)|(?P<n2>\d*\.\d+)([Ee](?P<exp1>[+-]*?\d+))?(?P<float1>[fFlL]*)|(?P<n4>\d+\.\d*)([Ee](?P<exp2>[+-]*?\d+))?(?P<float2>[fFlL]*)|(?P<oct>0*)(?P<n0>\d+)(?P<qual2>[uUlL]*)""",r'L?"([^"\\]|\\.)*"',r'[a-zA-Z_]\w*',r'%:%:|<<=|>>=|\.\.\.|<<|<%|<:|<=|>>|>=|\+\+|\+=|--|->|-=|\*=|/=|%:|%=|%>|==|&&|&=|\|\||\|=|\^=|:>|!=|##|[\(\)\{\}\[\]<>\?\|\^\*\+&=:!#;,%/\-\?\~\.]',]
36
re_clexer=re.compile('|'.join(["(?P<%s>%s)"%(name,part)for name,part in zip(tok_types,exp_types)]),re.M)
37
accepted='a'
38
ignored='i'
39
undefined='u'
40
skipped='s'
41
def repl(m):
42
s=m.group(1)
43
if s:
44
return' '
45
return m.group(3)or''
46
def filter_comments(filename):
47
code=Utils.readf(filename)
48
if use_trigraphs:
49
for(a,b)in trig_def:code=code.split(a).join(b)
50
code=re_nl.sub('',code)
51
code=re_cpp.sub(repl,code)
52
return[(m.group(2),m.group(3))for m in re.finditer(re_lines,code)]
53
prec={}
54
ops=['* / %','+ -','<< >>','< <= >= >','== !=','& | ^','&& ||',',']
55
for x in range(len(ops)):
56
syms=ops[x]
57
for u in syms.split():
58
prec[u]=x
59
def trimquotes(s):
60
if not s:return''
61
s=s.rstrip()
62
if s[0]=="'"and s[-1]=="'":return s[1:-1]
63
return s
64
def reduce_nums(val_1,val_2,val_op):
65
try:a=0+val_1
66
except TypeError:a=int(val_1)
67
try:b=0+val_2
68
except TypeError:b=int(val_2)
69
d=val_op
70
if d=='%':c=a%b
71
elif d=='+':c=a+b
72
elif d=='-':c=a-b
73
elif d=='*':c=a*b
74
elif d=='/':c=a/b
75
elif d=='^':c=a^b
76
elif d=='|':c=a|b
77
elif d=='||':c=int(a or b)
78
elif d=='&':c=a&b
79
elif d=='&&':c=int(a and b)
80
elif d=='==':c=int(a==b)
81
elif d=='!=':c=int(a!=b)
82
elif d=='<=':c=int(a<=b)
83
elif d=='<':c=int(a<b)
84
elif d=='>':c=int(a>b)
85
elif d=='>=':c=int(a>=b)
86
elif d=='^':c=int(a^b)
87
elif d=='<<':c=a<<b
88
elif d=='>>':c=a>>b
89
else:c=0
90
return c
91
def get_num(lst):
92
if not lst:raise PreprocError("empty list for get_num")
93
(p,v)=lst[0]
94
if p==OP:
95
if v=='(':
96
count_par=1
97
i=1
98
while i<len(lst):
99
(p,v)=lst[i]
100
if p==OP:
101
if v==')':
102
count_par-=1
103
if count_par==0:
104
break
105
elif v=='(':
106
count_par+=1
107
i+=1
108
else:
109
raise PreprocError("rparen expected %r"%lst)
110
(num,_)=get_term(lst[1:i])
111
return(num,lst[i+1:])
112
elif v=='+':
113
return get_num(lst[1:])
114
elif v=='-':
115
num,lst=get_num(lst[1:])
116
return(reduce_nums('-1',num,'*'),lst)
117
elif v=='!':
118
num,lst=get_num(lst[1:])
119
return(int(not int(num)),lst)
120
elif v=='~':
121
return(~int(num),lst)
122
else:
123
raise PreprocError("Invalid op token %r for get_num"%lst)
124
elif p==NUM:
125
return v,lst[1:]
126
elif p==IDENT:
127
return 0,lst[1:]
128
else:
129
raise PreprocError("Invalid token %r for get_num"%lst)
130
def get_term(lst):
131
if not lst:raise PreprocError("empty list for get_term")
132
num,lst=get_num(lst)
133
if not lst:
134
return(num,[])
135
(p,v)=lst[0]
136
if p==OP:
137
if v=='&&'and not num:
138
return(num,[])
139
elif v=='||'and num:
140
return(num,[])
141
elif v==',':
142
return get_term(lst[1:])
143
elif v=='?':
144
count_par=0
145
i=1
146
while i<len(lst):
147
(p,v)=lst[i]
148
if p==OP:
149
if v==')':
150
count_par-=1
151
elif v=='(':
152
count_par+=1
153
elif v==':':
154
if count_par==0:
155
break
156
i+=1
157
else:
158
raise PreprocError("rparen expected %r"%lst)
159
if int(num):
160
return get_term(lst[1:i])
161
else:
162
return get_term(lst[i+1:])
163
else:
164
num2,lst=get_num(lst[1:])
165
if not lst:
166
num2=reduce_nums(num,num2,v)
167
return get_term([(NUM,num2)]+lst)
168
p2,v2=lst[0]
169
if p2!=OP:
170
raise PreprocError("op expected %r"%lst)
171
if prec[v2]>=prec[v]:
172
num2=reduce_nums(num,num2,v)
173
return get_term([(NUM,num2)]+lst)
174
else:
175
num3,lst=get_num(lst[1:])
176
num3=reduce_nums(num2,num3,v2)
177
return get_term([(NUM,num),(p,v),(NUM,num3)]+lst)
178
raise PreprocError("cannot reduce %r"%lst)
179
def reduce_eval(lst):
180
num,lst=get_term(lst)
181
return(NUM,num)
182
def stringize(lst):
183
lst=[str(v2)for(p2,v2)in lst]
184
return"".join(lst)
185
def paste_tokens(t1,t2):
186
p1=None
187
if t1[0]==OP and t2[0]==OP:
188
p1=OP
189
elif t1[0]==IDENT and(t2[0]==IDENT or t2[0]==NUM):
190
p1=IDENT
191
elif t1[0]==NUM and t2[0]==NUM:
192
p1=NUM
193
if not p1:
194
raise PreprocError('tokens do not make a valid paste %r and %r'%(t1,t2))
195
return(p1,t1[1]+t2[1])
196
def reduce_tokens(lst,defs,ban=[]):
197
i=0
198
while i<len(lst):
199
(p,v)=lst[i]
200
if p==IDENT and v=="defined":
201
del lst[i]
202
if i<len(lst):
203
(p2,v2)=lst[i]
204
if p2==IDENT:
205
if v2 in defs:
206
lst[i]=(NUM,1)
207
else:
208
lst[i]=(NUM,0)
209
elif p2==OP and v2=='(':
210
del lst[i]
211
(p2,v2)=lst[i]
212
del lst[i]
213
if v2 in defs:
214
lst[i]=(NUM,1)
215
else:
216
lst[i]=(NUM,0)
217
else:
218
raise PreprocError("Invalid define expression %r"%lst)
219
elif p==IDENT and v in defs:
220
if isinstance(defs[v],str):
221
a,b=extract_macro(defs[v])
222
defs[v]=b
223
macro_def=defs[v]
224
to_add=macro_def[1]
225
if isinstance(macro_def[0],list):
226
del lst[i]
227
for x in range(len(to_add)):
228
lst.insert(i,to_add[x])
229
i+=1
230
else:
231
args=[]
232
del lst[i]
233
if i>=len(lst):
234
raise PreprocError("expected '(' after %r (got nothing)"%v)
235
(p2,v2)=lst[i]
236
if p2!=OP or v2!='(':
237
raise PreprocError("expected '(' after %r"%v)
238
del lst[i]
239
one_param=[]
240
count_paren=0
241
while i<len(lst):
242
p2,v2=lst[i]
243
del lst[i]
244
if p2==OP and count_paren==0:
245
if v2=='(':
246
one_param.append((p2,v2))
247
count_paren+=1
248
elif v2==')':
249
if one_param:args.append(one_param)
250
break
251
elif v2==',':
252
if not one_param:raise PreprocError("empty param in funcall %s"%p)
253
args.append(one_param)
254
one_param=[]
255
else:
256
one_param.append((p2,v2))
257
else:
258
one_param.append((p2,v2))
259
if v2=='(':count_paren+=1
260
elif v2==')':count_paren-=1
261
else:
262
raise PreprocError('malformed macro')
263
accu=[]
264
arg_table=macro_def[0]
265
j=0
266
while j<len(to_add):
267
(p2,v2)=to_add[j]
268
if p2==OP and v2=='#':
269
if j+1<len(to_add)and to_add[j+1][0]==IDENT and to_add[j+1][1]in arg_table:
270
toks=args[arg_table[to_add[j+1][1]]]
271
accu.append((STR,stringize(toks)))
272
j+=1
273
else:
274
accu.append((p2,v2))
275
elif p2==OP and v2=='##':
276
if accu and j+1<len(to_add):
277
t1=accu[-1]
278
if to_add[j+1][0]==IDENT and to_add[j+1][1]in arg_table:
279
toks=args[arg_table[to_add[j+1][1]]]
280
if toks:
281
accu[-1]=paste_tokens(t1,toks[0])
282
accu.extend(toks[1:])
283
else:
284
accu.append((p2,v2))
285
accu.extend(toks)
286
elif to_add[j+1][0]==IDENT and to_add[j+1][1]=='__VA_ARGS__':
287
va_toks=[]
288
st=len(macro_def[0])
289
pt=len(args)
290
for x in args[pt-st+1:]:
291
va_toks.extend(x)
292
va_toks.append((OP,','))
293
if va_toks:va_toks.pop()
294
if len(accu)>1:
295
(p3,v3)=accu[-1]
296
(p4,v4)=accu[-2]
297
if v3=='##':
298
accu.pop()
299
if v4==','and pt<st:
300
accu.pop()
301
accu+=va_toks
302
else:
303
accu[-1]=paste_tokens(t1,to_add[j+1])
304
j+=1
305
else:
306
accu.append((p2,v2))
307
elif p2==IDENT and v2 in arg_table:
308
toks=args[arg_table[v2]]
309
reduce_tokens(toks,defs,ban+[v])
310
accu.extend(toks)
311
else:
312
accu.append((p2,v2))
313
j+=1
314
reduce_tokens(accu,defs,ban+[v])
315
for x in range(len(accu)-1,-1,-1):
316
lst.insert(i,accu[x])
317
i+=1
318
def eval_macro(lst,defs):
319
reduce_tokens(lst,defs,[])
320
if not lst:raise PreprocError("missing tokens to evaluate")
321
(p,v)=reduce_eval(lst)
322
return int(v)!=0
323
def extract_macro(txt):
324
t=tokenize(txt)
325
if re_fun.search(txt):
326
p,name=t[0]
327
p,v=t[1]
328
if p!=OP:raise PreprocError("expected open parenthesis")
329
i=1
330
pindex=0
331
params={}
332
prev='('
333
while 1:
334
i+=1
335
p,v=t[i]
336
if prev=='(':
337
if p==IDENT:
338
params[v]=pindex
339
pindex+=1
340
prev=p
341
elif p==OP and v==')':
342
break
343
else:
344
raise PreprocError("unexpected token (3)")
345
elif prev==IDENT:
346
if p==OP and v==',':
347
prev=v
348
elif p==OP and v==')':
349
break
350
else:
351
raise PreprocError("comma or ... expected")
352
elif prev==',':
353
if p==IDENT:
354
params[v]=pindex
355
pindex+=1
356
prev=p
357
elif p==OP and v=='...':
358
raise PreprocError("not implemented (1)")
359
else:
360
raise PreprocError("comma or ... expected (2)")
361
elif prev=='...':
362
raise PreprocError("not implemented (2)")
363
else:
364
raise PreprocError("unexpected else")
365
return(name,[params,t[i+1:]])
366
else:
367
(p,v)=t[0]
368
return(v,[[],t[1:]])
369
re_include=re.compile('^\s*(<(?P<a>.*)>|"(?P<b>.*)")')
370
def extract_include(txt,defs):
371
m=re_include.search(txt)
372
if m:
373
if m.group('a'):return'<',m.group('a')
374
if m.group('b'):return'"',m.group('b')
375
toks=tokenize(txt)
376
reduce_tokens(toks,defs,['waf_include'])
377
if not toks:
378
raise PreprocError("could not parse include %s"%txt)
379
if len(toks)==1:
380
if toks[0][0]==STR:
381
return'"',toks[0][1]
382
else:
383
if toks[0][1]=='<'and toks[-1][1]=='>':
384
return stringize(toks).lstrip('<').rstrip('>')
385
raise PreprocError("could not parse include %s."%txt)
386
def parse_char(txt):
387
if not txt:raise PreprocError("attempted to parse a null char")
388
if txt[0]!='\\':
389
return ord(txt)
390
c=txt[1]
391
if c=='x':
392
if len(txt)==4 and txt[3]in string.hexdigits:return int(txt[2:],16)
393
return int(txt[2:],16)
394
elif c.isdigit():
395
if c=='0'and len(txt)==2:return 0
396
for i in 3,2,1:
397
if len(txt)>i and txt[1:1+i].isdigit():
398
return(1+i,int(txt[1:1+i],8))
399
else:
400
try:return chr_esc[c]
401
except KeyError:raise PreprocError("could not parse char literal '%s'"%txt)
402
def tokenize(s):
403
ret=[]
404
for match in re_clexer.finditer(s):
405
m=match.group
406
for name in tok_types:
407
v=m(name)
408
if v:
409
if name==IDENT:
410
try:v=g_optrans[v];name=OP
411
except KeyError:
412
if v.lower()=="true":
413
v=1
414
name=NUM
415
elif v.lower()=="false":
416
v=0
417
name=NUM
418
elif name==NUM:
419
if m('oct'):v=int(v,8)
420
elif m('hex'):v=int(m('hex'),16)
421
elif m('n0'):v=m('n0')
422
else:
423
v=m('char')
424
if v:v=parse_char(v)
425
else:v=m('n2')or m('n4')
426
elif name==OP:
427
if v=='%:':v='#'
428
elif v=='%:%:':v='##'
429
elif name==STR:
430
v=v[1:-1]
431
ret.append((name,v))
432
break
433
return ret
434
def define_name(line):
435
return re_mac.match(line).group(0)
436
class c_parser(object):
437
def __init__(self,nodepaths=None,defines=None):
438
self.lines=[]
439
if defines is None:
440
self.defs={}
441
else:
442
self.defs=dict(defines)
443
self.state=[]
444
self.count_files=0
445
self.currentnode_stack=[]
446
self.nodepaths=nodepaths or[]
447
self.nodes=[]
448
self.names=[]
449
self.curfile=''
450
self.ban_includes=set([])
451
def cached_find_resource(self,node,filename):
452
try:
453
nd=node.ctx.cache_nd
454
except:
455
nd=node.ctx.cache_nd={}
456
tup=(node,filename)
457
try:
458
return nd[tup]
459
except KeyError:
460
ret=node.find_resource(filename)
461
if ret:
462
if getattr(ret,'children',None):
463
ret=None
464
elif ret.is_child_of(node.ctx.bldnode):
465
tmp=node.ctx.srcnode.search(ret.path_from(node.ctx.bldnode))
466
if tmp and getattr(tmp,'children',None):
467
ret=None
468
nd[tup]=ret
469
return ret
470
def tryfind(self,filename):
471
self.curfile=filename
472
found=self.cached_find_resource(self.currentnode_stack[-1],filename)
473
for n in self.nodepaths:
474
if found:
475
break
476
found=self.cached_find_resource(n,filename)
477
if found:
478
self.nodes.append(found)
479
if filename[-4:]!='.moc':
480
self.addlines(found)
481
else:
482
if not filename in self.names:
483
self.names.append(filename)
484
return found
485
def addlines(self,node):
486
self.currentnode_stack.append(node.parent)
487
filepath=node.abspath()
488
self.count_files+=1
489
if self.count_files>recursion_limit:
490
raise PreprocError("recursion limit exceeded")
491
pc=self.parse_cache
492
debug('preproc: reading file %r',filepath)
493
try:
494
lns=pc[filepath]
495
except KeyError:
496
pass
497
else:
498
self.lines.extend(lns)
499
return
500
try:
501
lines=filter_comments(filepath)
502
lines.append((POPFILE,''))
503
lines.reverse()
504
pc[filepath]=lines
505
self.lines.extend(lines)
506
except IOError:
507
raise PreprocError("could not read the file %s"%filepath)
508
except Exception:
509
if Logs.verbose>0:
510
error("parsing %s failed"%filepath)
511
traceback.print_exc()
512
def start(self,node,env):
513
debug('preproc: scanning %s (in %s)',node.name,node.parent.name)
514
bld=node.ctx
515
try:
516
self.parse_cache=bld.parse_cache
517
except AttributeError:
518
bld.parse_cache={}
519
self.parse_cache=bld.parse_cache
520
self.addlines(node)
521
if env['DEFINES']:
522
try:
523
lst=['%s %s'%(x[0],trimquotes('='.join(x[1:])))for x in[y.split('=')for y in env['DEFINES']]]
524
lst.reverse()
525
self.lines.extend([('define',x)for x in lst])
526
except AttributeError:
527
pass
528
while self.lines:
529
(token,line)=self.lines.pop()
530
if token==POPFILE:
531
self.count_files-=1
532
self.currentnode_stack.pop()
533
continue
534
try:
535
ve=Logs.verbose
536
if ve:debug('preproc: line is %s - %s state is %s',token,line,self.state)
537
state=self.state
538
if token[:2]=='if':
539
state.append(undefined)
540
elif token=='endif':
541
state.pop()
542
if token[0]!='e':
543
if skipped in self.state or ignored in self.state:
544
continue
545
if token=='if':
546
ret=eval_macro(tokenize(line),self.defs)
547
if ret:state[-1]=accepted
548
else:state[-1]=ignored
549
elif token=='ifdef':
550
m=re_mac.match(line)
551
if m and m.group(0)in self.defs:state[-1]=accepted
552
else:state[-1]=ignored
553
elif token=='ifndef':
554
m=re_mac.match(line)
555
if m and m.group(0)in self.defs:state[-1]=ignored
556
else:state[-1]=accepted
557
elif token=='include'or token=='import':
558
(kind,inc)=extract_include(line,self.defs)
559
if inc in self.ban_includes:
560
continue
561
if token=='import':self.ban_includes.add(inc)
562
if ve:debug('preproc: include found %s (%s) ',inc,kind)
563
if kind=='"'or not strict_quotes:
564
self.tryfind(inc)
565
elif token=='elif':
566
if state[-1]==accepted:
567
state[-1]=skipped
568
elif state[-1]==ignored:
569
if eval_macro(tokenize(line),self.defs):
570
state[-1]=accepted
571
elif token=='else':
572
if state[-1]==accepted:state[-1]=skipped
573
elif state[-1]==ignored:state[-1]=accepted
574
elif token=='define':
575
try:
576
self.defs[define_name(line)]=line
577
except:
578
raise PreprocError("Invalid define line %s"%line)
579
elif token=='undef':
580
m=re_mac.match(line)
581
if m and m.group(0)in self.defs:
582
self.defs.__delitem__(m.group(0))
583
elif token=='pragma':
584
if re_pragma_once.match(line.lower()):
585
self.ban_includes.add(self.curfile)
586
except Exception ,e:
587
if Logs.verbose:
588
debug('preproc: line parsing failed (%s): %s %s',e,line,Utils.ex_stack())
589
def scan(task):
590
global go_absolute
591
try:
592
incn=task.generator.includes_nodes
593
except AttributeError:
594
raise Errors.WafError('%r is missing a feature such as "c", "cxx" or "includes": '%task.generator)
595
if go_absolute:
596
nodepaths=incn
597
else:
598
nodepaths=[x for x in incn if x.is_child_of(x.ctx.srcnode)or x.is_child_of(x.ctx.bldnode)]
599
tmp=c_parser(nodepaths)
600
tmp.start(task.inputs[0],task.env)
601
if Logs.verbose:
602
debug('deps: deps for %r: %r; unresolved %r'%(task.inputs,tmp.nodes,tmp.names))
603
return(tmp.nodes,tmp.names)
604
605
Utils.run_once(tokenize)
606
Utils.run_once(define_name)
b'\\ No newline at end of file'
Loggerhead is a web-based interface for Breezy
Version: 2.0.1