« back to all changes in this revision
Viewing changes to wafadmin/Tools/preproc.py
- browse files at revision 35
- compare with another revision
- download diff
- download tarball
- view history from revision 35
- Committer: Sergey Kryazhevskikh
- Date: 2015-11-24 10:33:37 UTC
- mfrom: (33.1.6 hamster-applet)
- Revision ID: soliverr@gmail.com-20151124103337-stu5r5o62leyo1eu
Merged with upstream in lp:~menesis/ubuntu/vivid/hamster-applet/vivid branch
- files added:
- .pc
- help/fa
- help/pl
- help/ru
- help/te
- wafadmin
- wafadmin/3rdpartys
- wafadmin/Tools
- files removed:
- data/hamster-applet.gschema.xml
- files modified:
- .gitignore
added
removed
wafadmin/Tools/preproc.py
1
#! /usr/bin/env python
2
# encoding: utf-8
3
4
import re,sys,os,string
5
import Logs,Build,Utils
6
from Logs import debug,error
7
import traceback
8
class PreprocError(Utils.WafError):
9
pass
10
POPFILE='-'
11
recursion_limit=100
12
go_absolute=0
13
standard_includes=['/usr/include']
14
if sys.platform=="win32":
15
standard_includes=[]
16
use_trigraphs=0
17
'apply the trigraph rules first'
18
strict_quotes=0
19
g_optrans={'not':'!','and':'&&','bitand':'&','and_eq':'&=','or':'||','bitor':'|','or_eq':'|=','xor':'^','xor_eq':'^=','compl':'~',}
20
re_lines=re.compile('^[ \t]*(#|%:)[ \t]*(ifdef|ifndef|if|else|elif|endif|include|import|define|undef|pragma)[ \t]*(.*)\r*$',re.IGNORECASE|re.MULTILINE)
21
re_mac=re.compile("^[a-zA-Z_]\w*")
22
re_fun=re.compile('^[a-zA-Z_][a-zA-Z0-9_]*[(]')
23
re_pragma_once=re.compile('^\s*once\s*',re.IGNORECASE)
24
re_nl=re.compile('\\\\\r*\n',re.MULTILINE)
25
re_cpp=re.compile(r"""(/\*[^*]*\*+([^/*][^*]*\*+)*/)|//[^\n]*|("(\\.|[^"\\])*"|'(\\.|[^'\\])*'|.[^/"'\\]*)""",re.MULTILINE)
26
trig_def=[('??'+a,b)for a,b in zip("=-/!'()<>",r'#~\|^[]{}')]
27
chr_esc={'0':0,'a':7,'b':8,'t':9,'n':10,'f':11,'v':12,'r':13,'\\':92,"'":39}
28
NUM='i'
29
OP='O'
30
IDENT='T'
31
STR='s'
32
CHAR='c'
33
tok_types=[NUM,STR,IDENT,OP]
34
exp_types=[r"""0[xX](?P<hex>[a-fA-F0-9]+)(?P<qual1>[uUlL]*)|L*?'(?P<char>(\\.|[^\\'])+)'|(?P<n1>\d+)[Ee](?P<exp0>[+-]*?\d+)(?P<float0>[fFlL]*)|(?P<n2>\d*\.\d+)([Ee](?P<exp1>[+-]*?\d+))?(?P<float1>[fFlL]*)|(?P<n4>\d+\.\d*)([Ee](?P<exp2>[+-]*?\d+))?(?P<float2>[fFlL]*)|(?P<oct>0*)(?P<n0>\d+)(?P<qual2>[uUlL]*)""",r'L?"([^"\\]|\\.)*"',r'[a-zA-Z_]\w*',r'%:%:|<<=|>>=|\.\.\.|<<|<%|<:|<=|>>|>=|\+\+|\+=|--|->|-=|\*=|/=|%:|%=|%>|==|&&|&=|\|\||\|=|\^=|:>|!=|##|[\(\)\{\}\[\]<>\?\|\^\*\+&=:!#;,%/\-\?\~\.]',]
35
re_clexer=re.compile('|'.join(["(?P<%s>%s)"%(name,part)for name,part in zip(tok_types,exp_types)]),re.M)
36
accepted='a'
37
ignored='i'
38
undefined='u'
39
skipped='s'
40
def repl(m):
41
s=m.group(1)
42
if s is not None:return' '
43
s=m.group(3)
44
if s is None:return''
45
return s
46
def filter_comments(filename):
47
code=Utils.readf(filename)
48
if use_trigraphs:
49
for(a,b)in trig_def:code=code.split(a).join(b)
50
code=re_nl.sub('',code)
51
code=re_cpp.sub(repl,code)
52
return[(m.group(2),m.group(3))for m in re.finditer(re_lines,code)]
53
prec={}
54
ops=['* / %','+ -','<< >>','< <= >= >','== !=','& | ^','&& ||',',']
55
for x in range(len(ops)):
56
syms=ops[x]
57
for u in syms.split():
58
prec[u]=x
59
def reduce_nums(val_1,val_2,val_op):
60
try:a=0+val_1
61
except TypeError:a=int(val_1)
62
try:b=0+val_2
63
except TypeError:b=int(val_2)
64
d=val_op
65
if d=='%':c=a%b
66
elif d=='+':c=a+b
67
elif d=='-':c=a-b
68
elif d=='*':c=a*b
69
elif d=='/':c=a/b
70
elif d=='^':c=a^b
71
elif d=='|':c=a|b
72
elif d=='||':c=int(a or b)
73
elif d=='&':c=a&b
74
elif d=='&&':c=int(a and b)
75
elif d=='==':c=int(a==b)
76
elif d=='!=':c=int(a!=b)
77
elif d=='<=':c=int(a<=b)
78
elif d=='<':c=int(a<b)
79
elif d=='>':c=int(a>b)
80
elif d=='>=':c=int(a>=b)
81
elif d=='^':c=int(a^b)
82
elif d=='<<':c=a<<b
83
elif d=='>>':c=a>>b
84
else:c=0
85
return c
86
def get_num(lst):
87
if not lst:raise PreprocError("empty list for get_num")
88
(p,v)=lst[0]
89
if p==OP:
90
if v=='(':
91
count_par=1
92
i=1
93
while i<len(lst):
94
(p,v)=lst[i]
95
if p==OP:
96
if v==')':
97
count_par-=1
98
if count_par==0:
99
break
100
elif v=='(':
101
count_par+=1
102
i+=1
103
else:
104
raise PreprocError("rparen expected %r"%lst)
105
(num,_)=get_term(lst[1:i])
106
return(num,lst[i+1:])
107
elif v=='+':
108
return get_num(lst[1:])
109
elif v=='-':
110
num,lst=get_num(lst[1:])
111
return(reduce_nums('-1',num,'*'),lst)
112
elif v=='!':
113
num,lst=get_num(lst[1:])
114
return(int(not int(num)),lst)
115
elif v=='~':
116
return(~int(num),lst)
117
else:
118
raise PreprocError("invalid op token %r for get_num"%lst)
119
elif p==NUM:
120
return v,lst[1:]
121
elif p==IDENT:
122
return 0,lst[1:]
123
else:
124
raise PreprocError("invalid token %r for get_num"%lst)
125
def get_term(lst):
126
if not lst:raise PreprocError("empty list for get_term")
127
num,lst=get_num(lst)
128
if not lst:
129
return(num,[])
130
(p,v)=lst[0]
131
if p==OP:
132
if v=='&&'and not num:
133
return(num,[])
134
elif v=='||'and num:
135
return(num,[])
136
elif v==',':
137
return get_term(lst[1:])
138
elif v=='?':
139
count_par=0
140
i=1
141
while i<len(lst):
142
(p,v)=lst[i]
143
if p==OP:
144
if v==')':
145
count_par-=1
146
elif v=='(':
147
count_par+=1
148
elif v==':':
149
if count_par==0:
150
break
151
i+=1
152
else:
153
raise PreprocError("rparen expected %r"%lst)
154
if int(num):
155
return get_term(lst[1:i])
156
else:
157
return get_term(lst[i+1:])
158
else:
159
num2,lst=get_num(lst[1:])
160
if not lst:
161
num2=reduce_nums(num,num2,v)
162
return get_term([(NUM,num2)]+lst)
163
p2,v2=lst[0]
164
if p2!=OP:
165
raise PreprocError("op expected %r"%lst)
166
if prec[v2]>=prec[v]:
167
num2=reduce_nums(num,num2,v)
168
return get_term([(NUM,num2)]+lst)
169
else:
170
num3,lst=get_num(lst[1:])
171
num3=reduce_nums(num2,num3,v2)
172
return get_term([(NUM,num),(p,v),(NUM,num3)]+lst)
173
raise PreprocError("cannot reduce %r"%lst)
174
def reduce_eval(lst):
175
num,lst=get_term(lst)
176
return(NUM,num)
177
def stringize(lst):
178
lst=[str(v2)for(p2,v2)in lst]
179
return"".join(lst)
180
def paste_tokens(t1,t2):
181
p1=None
182
if t1[0]==OP and t2[0]==OP:
183
p1=OP
184
elif t1[0]==IDENT and(t2[0]==IDENT or t2[0]==NUM):
185
p1=IDENT
186
elif t1[0]==NUM and t2[0]==NUM:
187
p1=NUM
188
if not p1:
189
raise PreprocError('tokens do not make a valid paste %r and %r'%(t1,t2))
190
return(p1,t1[1]+t2[1])
191
def reduce_tokens(lst,defs,ban=[]):
192
i=0
193
while i<len(lst):
194
(p,v)=lst[i]
195
if p==IDENT and v=="defined":
196
del lst[i]
197
if i<len(lst):
198
(p2,v2)=lst[i]
199
if p2==IDENT:
200
if v2 in defs:
201
lst[i]=(NUM,1)
202
else:
203
lst[i]=(NUM,0)
204
elif p2==OP and v2=='(':
205
del lst[i]
206
(p2,v2)=lst[i]
207
del lst[i]
208
if v2 in defs:
209
lst[i]=(NUM,1)
210
else:
211
lst[i]=(NUM,0)
212
else:
213
raise PreprocError("invalid define expression %r"%lst)
214
elif p==IDENT and v in defs:
215
if isinstance(defs[v],str):
216
a,b=extract_macro(defs[v])
217
defs[v]=b
218
macro_def=defs[v]
219
to_add=macro_def[1]
220
if isinstance(macro_def[0],list):
221
del lst[i]
222
for x in xrange(len(to_add)):
223
lst.insert(i,to_add[x])
224
i+=1
225
else:
226
args=[]
227
del lst[i]
228
if i>=len(lst):
229
raise PreprocError("expected '(' after %r (got nothing)"%v)
230
(p2,v2)=lst[i]
231
if p2!=OP or v2!='(':
232
raise PreprocError("expected '(' after %r"%v)
233
del lst[i]
234
one_param=[]
235
count_paren=0
236
while i<len(lst):
237
p2,v2=lst[i]
238
del lst[i]
239
if p2==OP and count_paren==0:
240
if v2=='(':
241
one_param.append((p2,v2))
242
count_paren+=1
243
elif v2==')':
244
if one_param:args.append(one_param)
245
break
246
elif v2==',':
247
if not one_param:raise PreprocError("empty param in funcall %s"%p)
248
args.append(one_param)
249
one_param=[]
250
else:
251
one_param.append((p2,v2))
252
else:
253
one_param.append((p2,v2))
254
if v2=='(':count_paren+=1
255
elif v2==')':count_paren-=1
256
else:
257
raise PreprocError('malformed macro')
258
accu=[]
259
arg_table=macro_def[0]
260
j=0
261
while j<len(to_add):
262
(p2,v2)=to_add[j]
263
if p2==OP and v2=='#':
264
if j+1<len(to_add)and to_add[j+1][0]==IDENT and to_add[j+1][1]in arg_table:
265
toks=args[arg_table[to_add[j+1][1]]]
266
accu.append((STR,stringize(toks)))
267
j+=1
268
else:
269
accu.append((p2,v2))
270
elif p2==OP and v2=='##':
271
if accu and j+1<len(to_add):
272
t1=accu[-1]
273
if to_add[j+1][0]==IDENT and to_add[j+1][1]in arg_table:
274
toks=args[arg_table[to_add[j+1][1]]]
275
if toks:
276
accu[-1]=paste_tokens(t1,toks[0])
277
accu.extend(toks[1:])
278
else:
279
accu.append((p2,v2))
280
accu.extend(toks)
281
elif to_add[j+1][0]==IDENT and to_add[j+1][1]=='__VA_ARGS__':
282
va_toks=[]
283
st=len(macro_def[0])
284
pt=len(args)
285
for x in args[pt-st+1:]:
286
va_toks.extend(x)
287
va_toks.append((OP,','))
288
if va_toks:va_toks.pop()
289
if len(accu)>1:
290
(p3,v3)=accu[-1]
291
(p4,v4)=accu[-2]
292
if v3=='##':
293
accu.pop()
294
if v4==','and pt<st:
295
accu.pop()
296
accu+=va_toks
297
else:
298
accu[-1]=paste_tokens(t1,to_add[j+1])
299
j+=1
300
else:
301
accu.append((p2,v2))
302
elif p2==IDENT and v2 in arg_table:
303
toks=args[arg_table[v2]]
304
reduce_tokens(toks,defs,ban+[v])
305
accu.extend(toks)
306
else:
307
accu.append((p2,v2))
308
j+=1
309
reduce_tokens(accu,defs,ban+[v])
310
for x in xrange(len(accu)-1,-1,-1):
311
lst.insert(i,accu[x])
312
i+=1
313
def eval_macro(lst,adefs):
314
reduce_tokens(lst,adefs,[])
315
if not lst:raise PreprocError("missing tokens to evaluate")
316
(p,v)=reduce_eval(lst)
317
return int(v)!=0
318
def extract_macro(txt):
319
t=tokenize(txt)
320
if re_fun.search(txt):
321
p,name=t[0]
322
p,v=t[1]
323
if p!=OP:raise PreprocError("expected open parenthesis")
324
i=1
325
pindex=0
326
params={}
327
prev='('
328
while 1:
329
i+=1
330
p,v=t[i]
331
if prev=='(':
332
if p==IDENT:
333
params[v]=pindex
334
pindex+=1
335
prev=p
336
elif p==OP and v==')':
337
break
338
else:
339
raise PreprocError("unexpected token (3)")
340
elif prev==IDENT:
341
if p==OP and v==',':
342
prev=v
343
elif p==OP and v==')':
344
break
345
else:
346
raise PreprocError("comma or ... expected")
347
elif prev==',':
348
if p==IDENT:
349
params[v]=pindex
350
pindex+=1
351
prev=p
352
elif p==OP and v=='...':
353
raise PreprocError("not implemented (1)")
354
else:
355
raise PreprocError("comma or ... expected (2)")
356
elif prev=='...':
357
raise PreprocError("not implemented (2)")
358
else:
359
raise PreprocError("unexpected else")
360
return(name,[params,t[i+1:]])
361
else:
362
(p,v)=t[0]
363
return(v,[[],t[1:]])
364
re_include=re.compile('^\s*(<(?P<a>.*)>|"(?P<b>.*)")')
365
def extract_include(txt,defs):
366
m=re_include.search(txt)
367
if m:
368
if m.group('a'):return'<',m.group('a')
369
if m.group('b'):return'"',m.group('b')
370
toks=tokenize(txt)
371
reduce_tokens(toks,defs,['waf_include'])
372
if not toks:
373
raise PreprocError("could not parse include %s"%txt)
374
if len(toks)==1:
375
if toks[0][0]==STR:
376
return'"',toks[0][1]
377
else:
378
if toks[0][1]=='<'and toks[-1][1]=='>':
379
return stringize(toks).lstrip('<').rstrip('>')
380
raise PreprocError("could not parse include %s."%txt)
381
def parse_char(txt):
382
if not txt:raise PreprocError("attempted to parse a null char")
383
if txt[0]!='\\':
384
return ord(txt)
385
c=txt[1]
386
if c=='x':
387
if len(txt)==4 and txt[3]in string.hexdigits:return int(txt[2:],16)
388
return int(txt[2:],16)
389
elif c.isdigit():
390
if c=='0'and len(txt)==2:return 0
391
for i in 3,2,1:
392
if len(txt)>i and txt[1:1+i].isdigit():
393
return(1+i,int(txt[1:1+i],8))
394
else:
395
try:return chr_esc[c]
396
except KeyError:raise PreprocError("could not parse char literal '%s'"%txt)
397
def tokenize(s):
398
ret=[]
399
for match in re_clexer.finditer(s):
400
m=match.group
401
for name in tok_types:
402
v=m(name)
403
if v:
404
if name==IDENT:
405
try:v=g_optrans[v];name=OP
406
except KeyError:
407
if v.lower()=="true":
408
v=1
409
name=NUM
410
elif v.lower()=="false":
411
v=0
412
name=NUM
413
elif name==NUM:
414
if m('oct'):v=int(v,8)
415
elif m('hex'):v=int(m('hex'),16)
416
elif m('n0'):v=m('n0')
417
else:
418
v=m('char')
419
if v:v=parse_char(v)
420
else:v=m('n2')or m('n4')
421
elif name==OP:
422
if v=='%:':v='#'
423
elif v=='%:%:':v='##'
424
elif name==STR:
425
v=v[1:-1]
426
ret.append((name,v))
427
break
428
return ret
429
class c_parser(object):
430
def __init__(self,nodepaths=None,defines=None):
431
self.lines=[]
432
if defines is None:
433
self.defs={}
434
else:
435
self.defs=dict(defines)
436
self.state=[]
437
self.env=None
438
self.count_files=0
439
self.currentnode_stack=[]
440
self.nodepaths=nodepaths or[]
441
self.nodes=[]
442
self.names=[]
443
self.curfile=''
444
self.ban_includes=[]
445
def tryfind(self,filename):
446
self.curfile=filename
447
found=self.currentnode_stack[-1].find_resource(filename)
448
for n in self.nodepaths:
449
if found:
450
break
451
found=n.find_resource(filename)
452
if not found:
453
if not filename in self.names:
454
self.names.append(filename)
455
else:
456
self.nodes.append(found)
457
if filename[-4:]!='.moc':
458
self.addlines(found)
459
return found
460
def addlines(self,node):
461
self.currentnode_stack.append(node.parent)
462
filepath=node.abspath(self.env)
463
self.count_files+=1
464
if self.count_files>recursion_limit:raise PreprocError("recursion limit exceeded")
465
pc=self.parse_cache
466
debug('preproc: reading file %r',filepath)
467
try:
468
lns=pc[filepath]
469
except KeyError:
470
pass
471
else:
472
self.lines=lns+self.lines
473
return
474
try:
475
lines=filter_comments(filepath)
476
lines.append((POPFILE,''))
477
pc[filepath]=lines
478
self.lines=lines+self.lines
479
except IOError:
480
raise PreprocError("could not read the file %s"%filepath)
481
except Exception:
482
if Logs.verbose>0:
483
error("parsing %s failed"%filepath)
484
traceback.print_exc()
485
def start(self,node,env):
486
debug('preproc: scanning %s (in %s)',node.name,node.parent.name)
487
self.env=env
488
variant=node.variant(env)
489
bld=node.__class__.bld
490
try:
491
self.parse_cache=bld.parse_cache
492
except AttributeError:
493
bld.parse_cache={}
494
self.parse_cache=bld.parse_cache
495
self.addlines(node)
496
if env['DEFLINES']:
497
self.lines=[('define',x)for x in env['DEFLINES']]+self.lines
498
while self.lines:
499
(kind,line)=self.lines.pop(0)
500
if kind==POPFILE:
501
self.currentnode_stack.pop()
502
continue
503
try:
504
self.process_line(kind,line)
505
except Exception,e:
506
if Logs.verbose:
507
debug('preproc: line parsing failed (%s): %s %s',e,line,Utils.ex_stack())
508
def process_line(self,token,line):
509
ve=Logs.verbose
510
if ve:debug('preproc: line is %s - %s state is %s',token,line,self.state)
511
state=self.state
512
if token in['ifdef','ifndef','if']:
513
state.append(undefined)
514
elif token=='endif':
515
state.pop()
516
if not token in['else','elif','endif']:
517
if skipped in self.state or ignored in self.state:
518
return
519
if token=='if':
520
ret=eval_macro(tokenize(line),self.defs)
521
if ret:state[-1]=accepted
522
else:state[-1]=ignored
523
elif token=='ifdef':
524
m=re_mac.search(line)
525
if m and m.group(0)in self.defs:state[-1]=accepted
526
else:state[-1]=ignored
527
elif token=='ifndef':
528
m=re_mac.search(line)
529
if m and m.group(0)in self.defs:state[-1]=ignored
530
else:state[-1]=accepted
531
elif token=='include'or token=='import':
532
(kind,inc)=extract_include(line,self.defs)
533
if inc in self.ban_includes:return
534
if token=='import':self.ban_includes.append(inc)
535
if ve:debug('preproc: include found %s (%s) ',inc,kind)
536
if kind=='"'or not strict_quotes:
537
self.tryfind(inc)
538
elif token=='elif':
539
if state[-1]==accepted:
540
state[-1]=skipped
541
elif state[-1]==ignored:
542
if eval_macro(tokenize(line),self.defs):
543
state[-1]=accepted
544
elif token=='else':
545
if state[-1]==accepted:state[-1]=skipped
546
elif state[-1]==ignored:state[-1]=accepted
547
elif token=='define':
548
m=re_mac.search(line)
549
if m:
550
name=m.group(0)
551
if ve:debug('preproc: define %s %s',name,line)
552
self.defs[name]=line
553
else:
554
raise PreprocError("invalid define line %s"%line)
555
elif token=='undef':
556
m=re_mac.search(line)
557
if m and m.group(0)in self.defs:
558
self.defs.__delitem__(m.group(0))
559
elif token=='pragma':
560
if re_pragma_once.search(line.lower()):
561
self.ban_includes.append(self.curfile)
562
def get_deps(node,env,nodepaths=[]):
563
gruik=c_parser(nodepaths)
564
gruik.start(node,env)
565
return(gruik.nodes,gruik.names)
566
re_inc=re.compile('^[ \t]*(#|%:)[ \t]*(include)[ \t]*(.*)\r*$',re.IGNORECASE|re.MULTILINE)
567
def lines_includes(filename):
568
code=Utils.readf(filename)
569
if use_trigraphs:
570
for(a,b)in trig_def:code=code.split(a).join(b)
571
code=re_nl.sub('',code)
572
code=re_cpp.sub(repl,code)
573
return[(m.group(2),m.group(3))for m in re.finditer(re_inc,code)]
574
def get_deps_simple(node,env,nodepaths=[],defines={}):
575
nodes=[]
576
names=[]
577
def find_deps(node):
578
lst=lines_includes(node.abspath(env))
579
for(_,line)in lst:
580
(t,filename)=extract_include(line,defines)
581
if filename in names:
582
continue
583
if filename.endswith('.moc'):
584
names.append(filename)
585
found=None
586
for n in nodepaths:
587
if found:
588
break
589
found=n.find_resource(filename)
590
if not found:
591
if not filename in names:
592
names.append(filename)
593
elif not found in nodes:
594
nodes.append(found)
595
find_deps(node)
596
find_deps(node)
597
return(nodes,names)
598
Loggerhead is a web-based interface for Breezy
Version: 2.0.1