3
SCons string substitution.
8
# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 The SCons Foundation
10
# Permission is hereby granted, free of charge, to any person obtaining
11
# a copy of this software and associated documentation files (the
12
# "Software"), to deal in the Software without restriction, including
13
# without limitation the rights to use, copy, modify, merge, publish,
14
# distribute, sublicense, and/or sell copies of the Software, and to
15
# permit persons to whom the Software is furnished to do so, subject to
16
# the following conditions:
18
# The above copyright notice and this permission notice shall be included
19
# in all copies or substantial portions of the Software.
21
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
22
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
23
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
25
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
26
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
27
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29
__revision__ = "src/engine/SCons/Subst.py 2013/03/03 09:48:35 garyo"
36
from SCons.Util import is_String, is_Sequence
38
# Indexed by the SUBST_* constants below.
39
_strconv = [SCons.Util.to_String_for_subst,
40
SCons.Util.to_String_for_subst,
41
SCons.Util.to_String_for_signature]
45
AllowableExceptions = (IndexError, NameError)
47
def SetAllowableExceptions(*excepts):
48
global AllowableExceptions
49
AllowableExceptions = [_f for _f in excepts if _f]
51
def raise_exception(exception, target, s):
52
name = exception.__class__.__name__
53
msg = "%s `%s' trying to evaluate `%s'" % (name, exception, s)
55
raise SCons.Errors.BuildError(target[0], msg)
57
raise SCons.Errors.UserError(msg)
61
class Literal(object):
62
"""A wrapper for a string. If you use this object wrapped
63
around a string, then it will be interpreted as literal.
64
When passed to the command interpreter, all special
65
characters will be escaped."""
66
def __init__(self, lstr):
72
def escape(self, escape_func):
73
return escape_func(self.lstr)
75
def for_signature(self):
81
class SpecialAttrWrapper(object):
82
"""This is a wrapper for what we call a 'Node special attribute.'
83
This is any of the attributes of a Node that we can reference from
84
Environment variable substitution, such as $TARGET.abspath or
85
$SOURCES[1].filebase. We implement the same methods as Literal
86
so we can handle special characters, plus a for_signature method,
87
such that we can return some canonical string during signature
88
calculation to avoid unnecessary rebuilds."""
90
def __init__(self, lstr, for_signature=None):
91
"""The for_signature parameter, if supplied, will be the
92
canonical string we return from for_signature(). Else
93
we will simply return lstr."""
96
self.forsig = for_signature
103
def escape(self, escape_func):
104
return escape_func(self.lstr)
106
def for_signature(self):
109
def is_literal(self):
112
def quote_spaces(arg):
113
"""Generic function for putting double quotes around any string that
114
has white space in it."""
115
if ' ' in arg or '\t' in arg:
120
class CmdStringHolder(collections.UserString):
121
"""This is a special class used to hold strings generated by
122
scons_subst() and scons_subst_list(). It defines a special method
123
escape(). When passed a function with an escape algorithm for a
124
particular platform, it will return the contained string with the
125
proper escape sequences inserted.
127
def __init__(self, cmd, literal=None):
128
collections.UserString.__init__(self, cmd)
129
self.literal = literal
131
def is_literal(self):
134
def escape(self, escape_func, quote_func=quote_spaces):
135
"""Escape the string with the supplied function. The
136
function is expected to take an arbitrary string, then
137
return it with all special characters escaped and ready
138
for passing to the command interpreter.
140
After calling this function, the next call to str() will
141
return the escaped string.
144
if self.is_literal():
145
return escape_func(self.data)
146
elif ' ' in self.data or '\t' in self.data:
147
return quote_func(self.data)
151
def escape_list(mylist, escape_func):
152
"""Escape a list of arguments by running the specified escape_func
153
on every object in the list that has an escape() method."""
154
def escape(obj, escape_func=escape_func):
157
except AttributeError:
160
return e(escape_func)
161
return list(map(escape, mylist))
163
class NLWrapper(object):
164
"""A wrapper class that delays turning a list of sources or targets
165
into a NodeList until it's needed. The specified function supplied
166
when the object is initialized is responsible for turning raw nodes
167
into proxies that implement the special attributes like .abspath,
168
.source, etc. This way, we avoid creating those proxies just
169
"in case" someone is going to use $TARGET or the like, and only
170
go through the trouble if we really have to.
172
In practice, this might be a wash performance-wise, but it's a little
173
cleaner conceptually...
176
def __init__(self, list, func):
179
def _return_nodelist(self):
181
def _gen_nodelist(self):
185
elif not is_Sequence(mylist):
187
# The map(self.func) call is what actually turns
188
# a list into appropriate proxies.
189
self.nodelist = SCons.Util.NodeList(list(map(self.func, mylist)))
190
self._create_nodelist = self._return_nodelist
192
_create_nodelist = _gen_nodelist
195
class Targets_or_Sources(collections.UserList):
196
"""A class that implements $TARGETS or $SOURCES expansions by in turn
197
wrapping a NLWrapper. This class handles the different methods used
198
to access the list, calling the NLWrapper to create proxies on demand.
200
Note that we subclass collections.UserList purely so that the
201
is_Sequence() function will identify an object of this class as
202
a list during variable expansion. We're not really using any
203
collections.UserList methods in practice.
205
def __init__(self, nl):
207
def __getattr__(self, attr):
208
nl = self.nl._create_nodelist()
209
return getattr(nl, attr)
210
def __getitem__(self, i):
211
nl = self.nl._create_nodelist()
213
def __getslice__(self, i, j):
214
nl = self.nl._create_nodelist()
215
i = max(i, 0); j = max(j, 0)
218
nl = self.nl._create_nodelist()
221
nl = self.nl._create_nodelist()
224
class Target_or_Source(object):
225
"""A class that implements $TARGET or $SOURCE expansions by in turn
226
wrapping a NLWrapper. This class handles the different methods used
227
to access an individual proxy Node, calling the NLWrapper to create
230
def __init__(self, nl):
232
def __getattr__(self, attr):
233
nl = self.nl._create_nodelist()
237
# If there is nothing in the list, then we have no attributes to
238
# pass through, so raise AttributeError for everything.
239
raise AttributeError("NodeList has no attribute: %s" % attr)
240
return getattr(nl0, attr)
242
nl = self.nl._create_nodelist()
247
nl = self.nl._create_nodelist()
252
class NullNodeList(SCons.Util.NullSeq):
253
def __call__(self, *args, **kwargs): return ''
254
def __str__(self): return ''
256
NullNodesList = NullNodeList()
258
def subst_dict(target, source):
259
"""Create a dictionary for substitution of special
260
construction variables.
262
This translates the following special arguments:
264
target - the target (object or array of objects),
265
used to generate the TARGET and TARGETS
266
construction variables
268
source - the source (object or array of objects),
269
used to generate the SOURCES and SOURCE
270
construction variables
275
def get_tgt_subst_proxy(thing):
277
subst_proxy = thing.get_subst_proxy()
278
except AttributeError:
279
subst_proxy = thing # probably a string, just return it
281
tnl = NLWrapper(target, get_tgt_subst_proxy)
282
dict['TARGETS'] = Targets_or_Sources(tnl)
283
dict['TARGET'] = Target_or_Source(tnl)
285
# This is a total cheat, but hopefully this dictionary goes
286
# away soon anyway. We just let these expand to $TARGETS
287
# because that's "good enough" for the use of ToolSurrogates
288
# (see test/ToolSurrogate.py) to generate documentation.
289
dict['CHANGED_TARGETS'] = '$TARGETS'
290
dict['UNCHANGED_TARGETS'] = '$TARGETS'
292
dict['TARGETS'] = NullNodesList
293
dict['TARGET'] = NullNodesList
296
def get_src_subst_proxy(node):
299
except AttributeError:
304
return node.get_subst_proxy()
305
except AttributeError:
306
return node # probably a String, just return it
307
snl = NLWrapper(source, get_src_subst_proxy)
308
dict['SOURCES'] = Targets_or_Sources(snl)
309
dict['SOURCE'] = Target_or_Source(snl)
311
# This is a total cheat, but hopefully this dictionary goes
312
# away soon anyway. We just let these expand to $TARGETS
313
# because that's "good enough" for the use of ToolSurrogates
314
# (see test/ToolSurrogate.py) to generate documentation.
315
dict['CHANGED_SOURCES'] = '$SOURCES'
316
dict['UNCHANGED_SOURCES'] = '$SOURCES'
318
dict['SOURCES'] = NullNodesList
319
dict['SOURCE'] = NullNodesList
323
# Constants for the "mode" parameter to scons_subst_list() and
324
# scons_subst(). SUBST_RAW gives the raw command line. SUBST_CMD
325
# gives a command line suitable for passing to a shell. SUBST_SIG
326
# gives a command line appropriate for calculating the signature
327
# of a command line...if this changes, we should rebuild.
332
_rm = re.compile(r'\$[()]')
333
_remove = re.compile(r'\$\([^\$]*(\$[^\)][^\$]*)*\$\)')
335
# Indexed by the SUBST_* constants above.
336
_regex_remove = [ _rm, None, _remove ]
339
#return [ l for l in list if not l in ('$(', '$)') ]
340
return [l for l in list if not l in ('$(', '$)')]
342
def _remove_list(list):
344
do_append = result.append
347
do_append = lambda x: None
349
do_append = result.append
354
# Indexed by the SUBST_* constants above.
355
_list_remove = [ _rm_list, None, _remove_list ]
357
# Regular expressions for splitting strings and handling substitutions,
358
# for use by the scons_subst() and scons_subst_list() functions:
360
# The first expression compiled matches all of the $-introduced tokens
361
# that we need to process in some way, and is used for substitutions.
362
# The expressions it matches are:
367
# "$variable" [must begin with alphabetic or underscore]
370
# The second expression compiled is used for splitting strings into tokens
371
# to be processed, and it matches all of the tokens listed above, plus
372
# the following that affect how arguments do or don't get joined together:
375
# "non-white-space" [without any dollar signs]
376
# "$" [single dollar sign]
378
_dollar_exps_str = r'\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}'
379
_dollar_exps = re.compile(r'(%s)' % _dollar_exps_str)
380
_separate_args = re.compile(r'(%s|\s+|[^\s\$]+|\$)' % _dollar_exps_str)
382
# This regular expression is used to replace strings of multiple white
383
# space characters in the string result from the scons_subst() function.
384
_space_sep = re.compile(r'[\t ]+(?![^{]*})')
386
def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None):
387
"""Expand a string or list containing construction variable
390
This is the work-horse function for substitutions in file names
391
and the like. The companion scons_subst_list() function (below)
392
handles separating command lines into lists of arguments, so see
393
that function if that's what you're looking for.
395
if isinstance(strSubst, str) and strSubst.find('$') < 0:
398
class StringSubber(object):
399
"""A class to construct the results of a scons_subst() call.
401
This binds a specific construction environment, mode, target and
402
source with two methods (substitute() and expand()) that handle
405
def __init__(self, env, mode, conv, gvars):
411
def expand(self, s, lvars):
412
"""Expand a single "token" as necessary, returning an
413
appropriate string containing the expansion.
415
This handles expanding different types of things (strings,
416
lists, callables) appropriately. It calls the wrapper
417
substitute() method to re-expand things as necessary, so that
418
the results of expansions of side-by-side strings still get
419
re-evaluated separately, not smushed together.
424
except (IndexError, ValueError):
434
if key[0] == '{' or key.find('.') >= 0:
438
s = eval(key, self.gvars, lvars)
439
except KeyboardInterrupt:
442
if e.__class__ in AllowableExceptions:
444
raise_exception(e, lvars['TARGETS'], s)
448
elif key in self.gvars:
450
elif not NameError in AllowableExceptions:
451
raise_exception(NameError(key), lvars['TARGETS'], s)
455
# Before re-expanding the result, handle
456
# recursive expansion by copying the local
457
# variable dictionary and overwriting a null
458
# string for the value of the variable name
461
# This could potentially be optimized by only
462
# copying lvars when s contains more expansions,
463
# but lvars is usually supposed to be pretty
464
# small, and deeply nested variable expansions
465
# are probably more the exception than the norm,
466
# so it should be tolerable for now.
468
var = key.split('.')[0]
470
return self.substitute(s, lv)
472
def func(l, conv=self.conv, substitute=self.substitute, lvars=lvars):
473
return conv(substitute(l, lvars))
474
return list(map(func, s))
477
s = s(target=lvars['TARGETS'],
478
source=lvars['SOURCES'],
480
for_signature=(self.mode != SUBST_CMD))
482
# This probably indicates that it's a callable
483
# object that doesn't match our calling arguments
485
if self.mode == SUBST_RAW:
488
return self.substitute(s, lvars)
494
def substitute(self, args, lvars):
495
"""Substitute expansions in an argument or list of arguments.
497
This serves as a wrapper for splitting up a string into
500
if is_String(args) and not isinstance(args, CmdStringHolder):
501
args = str(args) # In case it's a UserString.
503
def sub_match(match):
504
return self.conv(self.expand(match.group(1), lvars))
505
result = _dollar_exps.sub(sub_match, args)
507
# If the internal conversion routine doesn't return
508
# strings (it could be overridden to return Nodes, for
509
# example), then the 1.5.2 re module will throw this
510
# exception. Back off to a slower, general-purpose
511
# algorithm that works for all data types.
512
args = _separate_args.findall(args)
515
result.append(self.conv(self.expand(a, lvars)))
519
result = ''.join(map(str, result))
522
return self.expand(args, lvars)
525
conv = _strconv[mode]
527
# Doing this every time is a bit of a waste, since the Executor
528
# has typically already populated the OverrideEnvironment with
529
# $TARGET/$SOURCE variables. We're keeping this (for now), though,
530
# because it supports existing behavior that allows us to call
531
# an Action directly with an arbitrary target+source pair, which
532
# we use in Tool/tex.py to handle calling $BIBTEX when necessary.
533
# If we dropped that behavior (or found another way to cover it),
534
# we could get rid of this call completely and just rely on the
535
# Executor setting the variables.
536
if 'TARGET' not in lvars:
537
d = subst_dict(target, source)
542
# We're (most likely) going to eval() things. If Python doesn't
543
# find a __builtins__ value in the global dictionary used for eval(),
544
# it copies the current global values for you. Avoid this by
545
# setting it explicitly and then deleting, so we don't pollute the
546
# construction environment Dictionary(ies) that are typically used
548
gvars['__builtins__'] = __builtins__
550
ss = StringSubber(env, mode, conv, gvars)
551
result = ss.substitute(strSubst, lvars)
554
del gvars['__builtins__']
558
if is_String(result):
559
# Remove $(-$) pairs and any stuff in between,
560
# if that's appropriate.
561
remove = _regex_remove[mode]
563
result = remove.sub('', result)
564
if mode != SUBST_RAW:
565
# Compress strings of white space characters into
567
result = _space_sep.sub(' ', result).strip()
568
elif is_Sequence(result):
569
remove = _list_remove[mode]
571
result = remove(result)
575
#Subst_List_Strings = {}
577
def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None):
578
"""Substitute construction variables in a string (or list or other
579
object) and separate the arguments into a command list.
581
The companion scons_subst() function (above) handles basic
582
substitutions within strings, so see that function instead
583
if that's what you're looking for.
586
# Subst_List_Strings[strSubst] = Subst_List_Strings[strSubst] + 1
588
# Subst_List_Strings[strSubst] = 1
590
# SCons.Debug.caller_trace(1)
591
class ListSubber(collections.UserList):
592
"""A class to construct the results of a scons_subst_list() call.
594
Like StringSubber, this class binds a specific construction
595
environment, mode, target and source with two methods
596
(substitute() and expand()) that handle the expansion.
598
In addition, however, this class is used to track the state of
599
the result(s) we're gathering so we can do the appropriate thing
600
whenever we have to append another word to the result--start a new
601
line, start a new word, append to the current word, etc. We do
602
this by setting the "append" attribute to the right method so
603
that our wrapper methods only need ever call ListSubber.append(),
604
and the rest of the object takes care of doing the right thing
607
def __init__(self, env, mode, conv, gvars):
608
collections.UserList.__init__(self, [])
614
if self.mode == SUBST_RAW:
615
self.add_strip = lambda x: self.append(x)
617
self.add_strip = lambda x: None
621
def expand(self, s, lvars, within_list):
622
"""Expand a single "token" as necessary, appending the
623
expansion to the current result.
625
This handles expanding different types of things (strings,
626
lists, callables) appropriately. It calls the wrapper
627
substitute() method to re-expand things as necessary, so that
628
the results of expansions of side-by-side strings still get
629
re-evaluated separately, not smushed together.
635
except (IndexError, ValueError):
644
self.open_strip('$(')
646
self.close_strip('$)')
649
if key[0] == '{' or key.find('.') >= 0:
653
s = eval(key, self.gvars, lvars)
654
except KeyboardInterrupt:
657
if e.__class__ in AllowableExceptions:
659
raise_exception(e, lvars['TARGETS'], s)
663
elif key in self.gvars:
665
elif not NameError in AllowableExceptions:
666
raise_exception(NameError(), lvars['TARGETS'], s)
670
# Before re-expanding the result, handle
671
# recursive expansion by copying the local
672
# variable dictionary and overwriting a null
673
# string for the value of the variable name
676
var = key.split('.')[0]
678
self.substitute(s, lv, 0)
682
self.substitute(a, lvars, 1)
686
s = s(target=lvars['TARGETS'],
687
source=lvars['SOURCES'],
689
for_signature=(self.mode != SUBST_CMD))
691
# This probably indicates that it's a callable
692
# object that doesn't match our calling arguments
694
if self.mode == SUBST_RAW:
698
self.substitute(s, lvars, within_list)
704
def substitute(self, args, lvars, within_list):
705
"""Substitute expansions in an argument or list of arguments.
707
This serves as a wrapper for splitting up a string into
711
if is_String(args) and not isinstance(args, CmdStringHolder):
712
args = str(args) # In case it's a UserString.
713
args = _separate_args.findall(args)
715
if a[0] in ' \t\n\r\f\v':
723
self.expand(a, lvars, within_list)
725
self.expand(args, lvars, within_list)
728
"""Arrange for the next word to start a new line. This
729
is like starting a new word, except that we have to append
730
another line to the result."""
731
collections.UserList.append(self, [])
735
"""Arrange for the next word to append to the end of the
736
current last word in the result."""
737
self.append = self.add_to_current_word
740
"""Arrange for the next word to start a new word."""
741
self.append = self.add_new_word
743
def add_to_current_word(self, x):
744
"""Append the string x to the end of the current last word
745
in the result. If that is not possible, then just add
746
it as a new word. Make sure the entire concatenated string
747
inherits the object attributes of x (in particular, the
748
escape function) by wrapping it as CmdStringHolder."""
750
if not self.in_strip or self.mode != SUBST_SIG:
752
current_word = self[-1][-1]
756
# All right, this is a hack and it should probably
757
# be refactored out of existence in the future.
758
# The issue is that we want to smoosh words together
759
# and make one file name that gets escaped if
760
# we're expanding something like foo$EXTENSION,
761
# but we don't want to smoosh them together if
762
# it's something like >$TARGET, because then we'll
763
# treat the '>' like it's part of the file name.
764
# So for now, just hard-code looking for the special
765
# command-line redirection characters...
767
last_char = str(current_word)[-1]
770
if last_char in '<>|':
775
# We used to treat a word appended to a literal
776
# as a literal itself, but this caused problems
777
# with interpreting quotes around space-separated
778
# targets on command lines. Removing this makes
779
# none of the "substantive" end-to-end tests fail,
780
# so we'll take this out but leave it commented
781
# for now in case there's a problem not covered
782
# by the test cases and we need to resurrect this.
783
#literal1 = self.literal(self[-1][-1])
784
#literal2 = self.literal(x)
787
#y = CmdStringHolder(y, literal1 or literal2)
788
y = CmdStringHolder(y, None)
791
def add_new_word(self, x):
792
if not self.in_strip or self.mode != SUBST_SIG:
793
literal = self.literal(x)
796
x = CmdStringHolder(x, literal)
798
self.append = self.add_to_current_word
800
def literal(self, x):
803
except AttributeError:
808
def open_strip(self, x):
809
"""Handle the "open strip" $( token."""
813
def close_strip(self, x):
814
"""Handle the "close strip" $) token."""
819
conv = _strconv[mode]
821
# Doing this every time is a bit of a waste, since the Executor
822
# has typically already populated the OverrideEnvironment with
823
# $TARGET/$SOURCE variables. We're keeping this (for now), though,
824
# because it supports existing behavior that allows us to call
825
# an Action directly with an arbitrary target+source pair, which
826
# we use in Tool/tex.py to handle calling $BIBTEX when necessary.
827
# If we dropped that behavior (or found another way to cover it),
828
# we could get rid of this call completely and just rely on the
829
# Executor setting the variables.
830
if 'TARGET' not in lvars:
831
d = subst_dict(target, source)
836
# We're (most likely) going to eval() things. If Python doesn't
837
# find a __builtins__ value in the global dictionary used for eval(),
838
# it copies the current global values for you. Avoid this by
839
# setting it explicitly and then deleting, so we don't pollute the
840
# construction environment Dictionary(ies) that are typically used
842
gvars['__builtins__'] = __builtins__
844
ls = ListSubber(env, mode, conv, gvars)
845
ls.substitute(strSubst, lvars, 0)
848
del gvars['__builtins__']
854
def scons_subst_once(strSubst, env, key):
855
"""Perform single (non-recursive) substitution of a single
856
construction variable keyword.
858
This is used when setting a variable when copying or overriding values
859
in an Environment. We want to capture (expand) the old value before
860
we override it, so people can do things like:
862
env2 = env.Clone(CCFLAGS = '$CCFLAGS -g')
864
We do this with some straightforward, brute-force code here...
866
if isinstance(strSubst, str) and strSubst.find('$') < 0:
869
matchlist = ['$' + key, '${' + key + '}']
870
val = env.get(key, '')
871
def sub_match(match, val=val, matchlist=matchlist):
876
return ' '.join(map(str, a))
880
if is_Sequence(strSubst):
891
result.append(_dollar_exps.sub(sub_match, arg))
895
elif is_String(strSubst):
896
return _dollar_exps.sub(sub_match, strSubst)
902
# indent-tabs-mode:nil
904
# vim: set expandtab tabstop=4 shiftwidth=4:
added
removed
scons/scons-local/SCons/Subst.py
