1
from peak.util.assembler import *
3
from core import class_or_type_of, call_thru, flatten
6
from codegen import SMIGenerator, ExprBuilder, Getitem, IfElse, Tuple
7
from peak.util.decorators import decorate, synchronized, decorate_assignment
8
from types import InstanceType, ClassType
9
from ast_builder import build, parse_expr
10
import inspect, new, codegen, parser
13
'IsInstance', 'IsSubclass', 'Truth', 'Identity', 'Comparison', 'priority',
14
'IndexedEngine', 'predicate_node_for', 'meta_function', 'expressionSignature',
18
def predicate_node_for(builder, expr, cases, remaining_exprs, memo):
19
"""Return a dispatch tree node argument appropriate for the expr"""
21
def value_check(val, (exact, ranges)):
28
(tl,th), node = ranges[mid]
35
raise AssertionError("Should never get here")
38
def IsInstance(expr, code=None):
39
if code is None: return expr,
40
return IsSubclass(class_or_type_of(expr), code)
42
_unpack = lambda c: c.UNPACK_SEQUENCE(2)
43
subclass_check = TryExcept(
45
Code.DUP_TOP, SMIGenerator.ARG, _unpack, Code.ROT_THREE,
46
Code.POP_TOP, Code.BINARY_SUBSCR, Code.ROT_TWO, Code.POP_TOP
47
]), [(Const(KeyError), Suite([
48
SMIGenerator.ARG, _unpack, Code.POP_TOP, Call(Code.ROT_TWO, (Pass,)),
53
def IsSubclass(expr, code=None):
54
if code is None: return expr,
55
code(expr, subclass_check)
57
identity_check = IfElse(
58
Getitem(SMIGenerator.ARG, Code.ROT_TWO),
59
Compare(Code.DUP_TOP, [('in', SMIGenerator.ARG)]),
60
Suite([Code.POP_TOP, Getitem(SMIGenerator.ARG, None)])
64
def Identity(expr, code=None):
65
if code is None: return expr,
66
code(Call(Const(id), (expr,), fold=False), identity_check)
69
def Comparison(expr, code=None):
70
if code is None: return expr,
71
code.LOAD_CONST(value_check)
72
Call(Pass, (expr, SMIGenerator.ARG), code=code)
75
def Truth(expr, code=None):
76
if code is None: return expr,
78
code(SMIGenerator.ARG); code.UNPACK_SEQUENCE(2)
79
code(expr, skip.JUMP_IF_TRUE, Code.ROT_THREE, skip, Code.POP_TOP,
80
Code.ROT_TWO, Code.POP_TOP)
84
"""An integer priority for manually resolving a rule ambiguity"""
85
when(implies, (priority, priority))(lambda p1,p2: p1>=p2)
87
class ExprBuilder(ExprBuilder):
88
"""Extended expression builder with support for meta-functions"""
90
def Backquote(self, expr):
91
raise SyntaxError("backquotes are not allowed in predicates")
95
def meta_function(*stub, **parsers):
96
"""Declare a meta-function and its argument parsers"""
98
def callback(frame, name, func, old_locals):
99
for name in inspect.getargs(func.func_code)[0]:
100
if not isinstance(name, basestring):
102
"Meta-functions cannot have packed-tuple arguments"
104
what = func, parsers, inspect.getargspec(func)
105
meta_functions[stub] = (
106
lambda builder, *args: apply_meta(builder, what, *args)
109
return decorate_assignment(callback)
111
def expressionSignature(expr):
112
"""Convert raw Python code into logical conditions"""
113
# Default is to simply test the truth of the expression
114
return Test(Truth(expr), Value(True))
116
when(expressionSignature, (Const,))(lambda expr: bool(expr.value))
118
when(expressionSignature, ((bool, Test, Signature, Disjunction),))
119
def pass_through(expr):
124
class CriteriaBuilder(ExprBuilder):
125
simplify_comparisons = True
127
def build_with(self, expr):
130
return build(self, expr)
135
return codegen.Not(self.build_with(expr))
138
return codegen.Or(map(self.build_with, items))
140
def CallFunc(self, func, args, kw, star_node, dstar_node):
141
b = build.__get__(self)
143
if isinstance(target, Const) and target.value in meta_functions:
144
return meta_functions[target.value](
145
self, args, kw, star_node, dstar_node
148
target, map(b,args), [(b(k),b(v)) for k,v in kw],
149
star_node and b(star_node), dstar_node and b(dstar_node)
152
def parse(self, expr):
153
return expressionSignature(ExprBuilder.parse(self, expr))
156
when(expressionSignature, codegen.And)
157
def do_intersect(expr):
158
return reduce(intersect, map(expressionSignature, expr.values), True)
160
when(expressionSignature, codegen.Or)
162
return OrElse(map(expressionSignature, expr.values))
165
when(expressionSignature, codegen.Not)
167
return negate(expressionSignature(expr.expr))
170
'>': '<', '>=': '<=', '=>':'<=',
171
'<': '>', '<=': '>=', '=<':'>=',
172
'<>': '<>', '!=': '<>', '==':'==',
173
'is': 'is', 'is not': 'is not'
176
when(expressionSignature, codegen.Compare)
177
def do_compare(expr):
179
(op, right), = expr.ops
181
if isinstance(left, Const) and op in _mirror_ops:
182
left, right, op = right, left, _mirror_ops[op]
184
if isinstance(right, Const):
185
if op=='in' or op=='not in':
186
cond = compileIn(left, right.value)
188
return maybe_invert(cond, op=='in')
189
elif op=='is' or op=='is not':
190
return maybe_invert(compileIs(left, right.value), op=='is')
192
return Test(Comparison(left), Inequality(op, right.value))
194
# Both sides involve variables or an un-optimizable constant,
195
# so it's a generic boolean criterion :(
196
return Test(Truth(expr), Value(True))
206
def apply_meta(builder,
207
(func, parsers, (argnames, varargs, varkw, defaults)), args, kw, star, dstar
209
# NB: tuple-args not allowed!
210
def parse(arg, node):
213
return parsers.get(arg, build)(builder, node)
218
for name in argnames:
219
if name=='__builder__': data[name] = builder
220
elif name=='__star__': data[name] = parse(name, star)
221
elif name=='__dstar__': data[name] = parse(name, dstar)
226
for k, v in zip(argnames[offset:], args):
227
data[k] = parse(k, v)
229
varargpos = len(argnames)-offset
230
if len(args)> varargpos:
232
raise TypeError("Too many arguments for %r" % (func,))
233
extra.extend([parse(varargs, node) for node in args[varargpos:]])
236
k = build(builder, k)
237
assert type(k) is Const and isinstance(k.value, basestring)
240
raise TypeError("Duplicate keyword %s for %r" % (k,func))
242
if varkw and k not in argnames and k not in parsers:
243
data[k] = parse(varkw, v)
245
data[k] = parse(k, v)
247
if star and '__star__' not in data:
248
raise TypeError("%r does not support parsing *args" % (func,))
250
if dstar and '__dstar__' not in data:
251
raise TypeError("%r does not support parsing **kw" % (func,))
254
for k,v in zip(argnames[-len(defaults):], defaults):
255
data.setdefault(k, v)
258
args = map(data.pop, argnames)+extra
261
"Missing positional argument %s for %r"%(e.args[0], func)
263
return func(*args, **data)
266
def compile_let(builder, args, kw, star, dstar):
267
"""Compile the let() function"""
268
if args or star or dstar:
269
raise TypeError("let() only accepts inline keyword arguments")
272
k = build(builder, k)
273
assert type(k) is Const and isinstance(k.value, basestring)
275
v = build(builder, v)
279
from peak.rules import let
280
meta_functions[let] = compile_let
288
def _expand_as(func, predicate_string, *namespaces):
289
"""Pre-parse predicate string and register meta function"""
291
args, varargs, kw, defaults = arginfo = inspect.getargspec(func)
292
argnames = list(flatten(filter(None, [args, varargs, kw])))
293
parsed = parser.expr(predicate_string).totuple(1)[1]
294
builder = CriteriaBuilder(
295
dict([(arg,Local(arg)) for arg in argnames]), *namespaces
298
for b in builder.bindings[-len(namespaces):][::-1]:
301
# Make a function that just gets the arguments we want
302
c = Code.from_function(func)
303
c.return_(Call(Const(locals),fold=False))
304
getargs = new.function(
305
c.code(), func.func_globals, func.func_name, func.func_defaults,
309
def expand(builder, *args):
310
builder.push(bindings) # globals, locals, etc.
311
builder.bind(apply_meta(builder, (getargs, {}, arginfo), *args))
313
# build in the newly-isolated namespace
314
result = build(builder, parsed)
318
meta_functions[func] = expand
320
c = Code.from_function(func)
322
if func.func_code.co_code == c.code().co_code: # function body is empty
323
c = Code.from_function(func)
324
c.return_(build(builder, parsed))
325
func.func_code = c.code()
329
def compileIs(expr, criterion):
330
"""Return a signature or predicate for 'expr is criterion'"""
331
#if criterion is None: # XXX this should be smarter
332
# return Test(IsInstance(expr), istype(NoneType))
334
return Test(Identity(expr), IsObject(criterion))
336
def maybe_invert(cond, truth):
337
if not truth: return negate(cond)
340
def compileIn(expr, criterion):
341
"""Return a signature or predicate (or None) for 'expr in criterion'"""
345
pass # treat the in condition as a truth expression
347
expr = Comparison(expr)
348
return Test(expr, Disjunction([Value(v) for v in criterion]))
350
when(compileIn, (object, (type, ClassType)))
351
def compileInClass(expr, criterion):
352
warn_parse("'x in SomeClass' syntax is deprecated; use 'isinstance(x,SomeClass)'")
353
return Test(IsInstance(expr), Class(criterion))
355
when(compileIn, (object, istype))
356
def compileInIsType(expr, criterion):
357
warn_parse("'x in istype(y)' syntax is deprecated; use 'type(x) is y'")
358
return Test(IsInstance(expr), criterion)
370
def warn_parse(message, category=DeprecationWarning):
371
"""Issue a warning about a parsed string"""
372
from warnings import warn, warn_explicit
374
# Find the original call to _parse_string() to get its ParseContext
376
frame = sys._getframe(3)
377
code = _parse_string.func_code
379
while frame is not None and frame.f_code is not code:
383
# XXX direct use of expressionSignature; can't pinpoint a location
384
return warn(message, category, ct)
385
ctx = frame.f_locals['ctx']
387
# Issue a warning against the method body
389
#lineno = getattr(getattr(ctx.body, 'func_code', None), 'co_firstlineno', 2)
390
module = g.get('__name__', "<string>")
391
filename = g.get('__file__')
393
fnl = filename.lower()
394
if fnl.endswith(".pyc") or fnl.endswith(".pyo"):
395
filename = filename[:-1]
397
if module == "__main__":
398
filename = sys.argv[0]
402
return warn_explicit(
403
message, category, filename, ctx.lineno,
404
g.setdefault("__warningregistry__", {})
411
class IndexedEngine(Engine, TreeBuilder):
412
"""A dispatching engine that builds trees using bitmap indexes"""
414
def __init__(self, disp):
417
super(IndexedEngine, self).__init__(disp)
418
self.arguments = dict([(arg,Local(arg)) for arg in self.argnames])
420
def _add_method(self, signature, rule):
421
signature = Signature(tests_for(signature, self))
422
if signature not in self.registry:
423
case_id = len(self.signatures)
424
self.signatures.append(signature)
426
exprs = self.all_exprs
427
for _t, expr, criterion in tests_for(signature, self):
428
Ordering(self, expr).requires(requires)
429
requires.append(expr)
430
index_type = bitmap_index_type(self, expr)
431
if index_type is not None:
432
if expr not in exprs:
434
if always_testable(expr):
435
Ordering(self, expr).requires([])
436
index_type(self, expr).add_case(case_id, criterion)
437
return super(IndexedEngine, self)._add_method(signature, rule)
439
def _generate_code(self):
440
smig = SMIGenerator(self.function)
441
all_exprs = map(self.to_expression, self.all_exprs)
442
for expr in all_exprs:
443
smig.maybe_cache(expr)
445
memo = dict([(expr, smig.action_id(expr)) for expr in all_exprs])
446
return smig.generate(self.build_root(memo)).func_code
448
def _full_reset(self):
449
# Replace the entire engine with a new one
450
Dispatching(self.function).create_engine(self.__class__)
453
def seed_bits(self, expr, cases):
454
return BitmapIndex(self, expr).seed_bits(cases)
457
def reseed(self, expr, criterion):
458
return BitmapIndex(self, expr).reseed(criterion)
460
# Make build() a synchronized method
461
build = synchronized(TreeBuilder.build.im_func)
463
def build_root(self, memo):
465
to_bits([len(self.signatures)])-1, frozenset(self.all_exprs), memo
468
def best_expr(self, cases, exprs):
469
return super(IndexedEngine, self).best_expr(
470
list(from_bits(cases)), exprs
473
def build_node(self, expr, cases, remaining_exprs, memo):
474
return memo[expr], predicate_node_for(
475
self, expr, cases, remaining_exprs, memo
478
def selectivity(self, expr, cases):
479
return BitmapIndex(self, expr).selectivity(cases)
482
def to_expression(self, expr):
493
def build_leaf(self, cases, memo):
494
action = self.rules.default_action
495
signatures = self.signatures
496
registry = self.registry
497
for case_no in from_bits(cases):
498
action = combine_actions(action, registry[signatures[case_no]])
499
# No need to memoize here, since the combined action probably isn't
500
# a meaningful key, and template-compiled methods are memoized at a
501
# lower level anyway.
502
return (0, compile_method(action, self))
504
when(bitmap_index_type, (IndexedEngine, Truth))(lambda en,ex:TruthIndex)
505
when(predicate_node_for, (IndexedEngine, Truth))
506
def truth_node(builder, expr, cases, remaining_exprs, memo):
507
dont_cares, seedmap = builder.seed_bits(expr, cases)
508
return ( # True/false tuple for Truth
509
builder.build(seedmap[True][0] | dont_cares, remaining_exprs, memo),
510
builder.build(seedmap[False][0] | dont_cares, remaining_exprs, memo)
513
when(bitmap_index_type, (IndexedEngine, Identity))(lambda en,ex:PointerIndex)
514
when(predicate_node_for, (IndexedEngine, Identity))
515
def identity_node(builder, expr, cases, remaining_exprs, memo):
516
dont_cares, seedmap = builder.seed_bits(expr, cases)
518
[(seed, builder.build(inc|dont_cares, remaining_exprs, memo))
519
for seed, (inc, exc) in seedmap.iteritems()]
522
when(bitmap_index_type, (IndexedEngine, Comparison))(lambda en,ex:RangeIndex)
523
when(predicate_node_for, (IndexedEngine, Comparison))
524
def range_node(builder, expr, cases, remaining_exprs, memo):
525
dontcares, seedmap = builder.seed_bits(expr, cases)
527
dontcares, seedmap, lambda cases: builder.build(cases, remaining_exprs, memo)
531
except NameError: from core import frozenset
534
when(bitmap_index_type, (IndexedEngine, type(None)))(value(None))
536
when(bitmap_index_type, (IndexedEngine, (IsInstance, IsSubclass)))(
540
when(predicate_node_for, (IndexedEngine, (IsInstance, IsSubclass)))
541
def class_node(builder, expr, cases, remaining_exprs, memo):
542
dontcares, seedmap = builder.seed_bits(expr, cases)
546
inc, exc = seedmap[cls]
548
builder.reseed(expr, Class(cls))
549
seedmap.update(builder.seed_bits(expr, cases)[1])
550
inc, exc = seedmap[cls]
551
cbits = dontcares | inc
552
cbits ^= (exc & cbits)
553
return cache.setdefault(cls, builder.build(cbits,remaining_exprs,memo))
555
return cache, lookup_fn
559
def type_to_test(typ, expr, engine):
560
"""Convert `typ` to a ``Test()`` of `expr` for `engine`"""
562
when(type_to_test, (type,))
563
when(type_to_test, (ClassType,))
564
def std_type_to_test(typ, expr, engine):
565
return Test(IsInstance(expr), Class(typ))
567
when(type_to_test, (istype,))
568
def istype_to_test(typ, expr, engine):
569
return Test(IsInstance(expr), typ)
575
when(tests_for, (istype(tuple), Engine))
576
def tests_for_tuple(ob, engine):
577
for cls, arg in zip(ob, engine.argnames):
578
yield type_to_test(cls, Local(arg), engine)
580
def always_testable(expr):
581
"""Is `expr` safe to evaluate in any order?"""
584
#when(always_testable, (IsSubclass,)) XXX might not be a class!
586
when(always_testable, ((Identity, Truth, Comparison, IsInstance),))
587
def testable_criterion(expr):
588
return always_testable(expr.expr)
590
when(always_testable, ((Local, Const),))(value(True))
593
when(parse_rule, (IndexedEngine, basestring))
594
def _parse_string(engine, predicate, ctx, cls):
595
b = CriteriaBuilder(engine.arguments, ctx.localdict, ctx.globaldict, __builtins__)
596
expr = b.parse(predicate)
597
bindings = b.bindings[0]
598
if cls is not None and engine.argnames:
599
cls = type_to_test(cls, engine.arguments[engine.argnames[0]], engine)
600
expr = intersect(cls, expr)
601
# XXX rewrap body for bindings here? problem: CSE isn't ready
602
# XXX so we'd have to make a temporary wrapper that gets replaced later. :(
603
# XXX (the wrapper would just always recalc the values)
604
# XXX Ugly bit at that point is that we're (slowly) generating code TWICE
606
maybe_bind(ctx.body, bindings), expr, ctx.actiontype, ctx.sequence
616
def maybe_bind(func, bindings):
617
"""Apply expression bindings to arguments, if applicable"""
619
if not bindings or not hasattr(func, 'func_code'):
620
return func # no bindings or not a function
622
args, varargs, varkw, defaults = inspect.getargspec(func)
623
if not args or isinstance(args[0], basestring):
624
return func # no args or first arg isn't a tuple
627
if not isinstance(arg, basestring): # nested tuple arg, not a binding
633
if arg not in bindings:
634
raise TypeError("Missing binding for %r" % arg)
637
return func # none of the tuple args are in the binding
639
argtuple = Tuple([bindings[arg] for arg in args[0]])
641
c = Code.from_spec(func.func_name, args[1:], varargs, varkw)
643
c.code(), func.func_globals, func.func_name, func.func_defaults
645
f.func_code = c.code() # make f's signature match func w/out tuple
646
c.return_(call_thru(f, Const(func), [argtuple])) # call to match that
647
f.func_code = c.code() # now include the actual body
648
f.__predicate_bindings__ = bindings, func # mark for later optimization
657
# === As of this point, it should be possible to compile expressions!
659
meta_function(isinstance)(
660
lambda __star__, __dstar__, *args, **kw:
661
compileIsXCall(isinstance, IsInstance, args, kw, __star__, __dstar__)
663
meta_function(issubclass)(
664
lambda __star__, __dstar__, *args, **kw:
665
compileIsXCall(issubclass, IsSubclass, args, kw, __star__, __dstar__)
667
def compileIsXCall(func, test, args, kw, star, dstar):
669
kw or len(args)!=2 or not isinstance(args[1], Const)
670
or isinstance(args[0], Const) or star is not None or dstar is not None
672
return expressionSignature(
673
Call(Const(func), args, tuple(kw.items()), star, dstar)
676
return Test(test(expr), Disjunction(map(Class, _yield_tuples(seq.value))))
678
def _yield_tuples(ob):
679
if type(ob) is tuple:
681
for i2 in _yield_tuples(i1):
687
# matches 'type(x) is y'
688
"isinstance(expr,Call) and isinstance(expr.func, Const)"
689
" and (expr.func.value is type) and len(expr.args)==1"
691
def compileTypeIsX(expr, criterion):
692
return Test(IsInstance(expr.args[0]), istype(criterion))
694
when(expressionSignature, "isinstance(expr, Const) and isinstance(expr.value,priority)")
695
def test_for_priority(expr):
696
return Test(None, expr.value)