« back to all changes in this revision
Viewing changes to system.py
- browse files at revision 9
- compare with another revision
- download diff
- download tarball
- view history from revision 9
- Committer: Sjoerd de Vries
- Date: 2009-01-29 01:36:20 UTC
- Revision ID: sjoerd@sjoerd-20090129013620-vhnc07kh3vr1yb57
forgotten files and metatxt files added
- files added:
- LICENSE.txt
added
removed
system.py
1
# Copyright 2007, Sjoerd de Vries
2
# This file is part of the Spyder main program (a.k.a the Spyder kernel)
3
# For licensing information, see LICENSE.txt
4
5
import os, zlib, re, sys, imp, compiler
6
from copy import copy, deepcopy
7
from Spydercompile import Spydercompile, validvar
8
9
commentmatch = re.compile(r'(?:^(?:.*?)##(?:.*?)$)', re.MULTILINE)
10
brackmatch = re.compile(r'\[[^\[\]]*?\]')
11
commamatch = re.compile(r'[ \n]*,[ \n]*([^ \n]|$)')
12
quotematch = re.compile(r'(?<![\\])(([\"\']).*?(?<![\\])\2)')
13
14
var = {}
15
alltypes = []
16
17
def changetempdir(tempdir):
18
if not os.path.exists(tempdir):
19
dirs = [a for a in os.path.abspath(tempdir).split(os.sep) if len(a) > 0]
20
currdir = ""
21
if dirs[0].endswith(":"):
22
currdir = dirs[0]
23
dirs = dirs[1:]
24
for d in dirs:
25
currdir += os.sep + d
26
if not os.path.exists(currdir): os.mkdir(currdir)
27
os.chdir(tempdir)
28
29
def dictparse(s,mask): #adapted from core/parse.py
30
if len(s) == 0: return {}
31
#mask all sub-lists
32
mask0 = mask
33
while 1:
34
p = brackmatch.finditer(mask0)
35
pos = 0
36
mask2 = ""
37
for pp in p:
38
mask2 += mask0[pos:pp.start()] + "!" * (pp.end()-pp.start())
39
pos = pp.end()
40
mask2 += mask0[pos:]
41
if pos == 0: break
42
mask0 = mask2
43
#split; we can't use split() because of the mask (we need start/end)
44
elements = []
45
p = commamatch.finditer(mask2)
46
pos = 0
47
for pp in p:
48
elements.append( [s[pos:pp.start()], mask[pos:pp.start()], mask2[pos:pp.start()]] )
49
pos = pp.end() - 1
50
s = s.rstrip()
51
if len(s[pos:].strip()) > 0: elements.append([s[pos:], mask[pos:len(s)], mask2[pos:len(s)]])
52
53
pairs = {}
54
for e in elements:
55
if e[0].startswith('\n'):
56
e[0] = e[0][len('\n'):].lstrip()
57
e[1] = e[1][len(e[1])-len(e[0]):]
58
e[2] = e[2][len(e[2])-len(e[0]):]
59
if len(e[0]) == 0: continue
60
v = e[0]
61
m = e[1]
62
m2 = e[2]
63
v =v.lstrip()
64
m = m[len(m)-len(v):]
65
m2 = m2[len(m2)-len(v):]
66
bm = m.find('[')
67
bm2 = m.rfind(']')
68
ee = m2.find(":")
69
if ee > -1: #pair
70
k = v[:ee].rstrip()
71
v = v[ee+1:]
72
m = m[ee+1:]
73
v =v.lstrip()
74
m = m[len(m)-len(v):]
75
if (k[0] != "\"" or k[-1] != "\"") and (k[0] != "'" or k[-1] != "'"):
76
raise SyntaxError("Malformed dictionary key %s" % k)
77
k = k[1:-1]
78
if k in pairs:
79
raise SyntaxError("Double key %s" % k)
80
bm = m.find('[')
81
bm2 = m.rfind(']')
82
83
if v[0] in ('[','('): #list
84
v = v[1:-1]
85
m = m[1:-1]
86
pairs[k] = listparse(v,m)
87
elif v[0] == '{': #dict
88
v = v[1:-1]
89
m = m[1:-1]
90
pairs[k] = dictparse(v,m)
91
elif bm > -1: #function + args
92
print s
93
print m
94
raise Exception
95
else: #value
96
assert (v[0] == v[-1] == "'") or (v[0] == v[-1] == '"')
97
pairs[k] = eval("str(%s)" % v)
98
else:
99
raise SyntaxError("Malformed dictionary pair %s" % e[0])
100
return pairs
101
102
def listparse(s,mask): #adapted from core/parse.py
103
if len(s) == 0: return []
104
#mask all sub-lists
105
mask0 = mask
106
while 1:
107
p = brackmatch.finditer(mask0)
108
pos = 0
109
mask2 = ""
110
for pp in p:
111
mask2 += mask0[pos:pp.start()] + "!" * (pp.end()-pp.start())
112
pos = pp.end()
113
mask2 += mask0[pos:]
114
if pos == 0: break
115
mask0 = mask2
116
#split; we can't use split() because of the mask (we need start/end)
117
elements = []
118
p = commamatch.finditer(mask2)
119
pos = 0
120
for pp in p:
121
elements.append( [s[pos:pp.start()], mask[pos:pp.start()], mask2[pos:pp.start()]] )
122
pos = pp.end() - 1
123
s = s.rstrip()
124
if len(s[pos:].strip()) > 0: elements.append([s[pos:], mask[pos:len(s)], mask2[pos:len(s)]])
125
126
values = []
127
for e in elements:
128
if e[0].startswith('\n'):
129
e[0] = e[0][len('\n'):].lstrip()
130
e[1] = e[1][len(e[1])-len(e[0]):]
131
if len(e[0]) == 0: continue
132
v = e[0]
133
m = e[1]
134
v =v.lstrip()
135
m = m[len(m)-len(v):]
136
bm = m.find('[')
137
bm2 = m.rfind(']')
138
if v[0] in ('[','('): #list
139
v = v[1:-1]
140
m = m[1:-1]
141
values.append(listparse(v,m))
142
elif v[0] == '{': #dict
143
v = v[1:-1]
144
m = m[1:-1]
145
values.append(dictparse(v,m))
146
elif bm > -1: #function + args
147
raise Exception
148
else: #value
149
assert v[0] == v[-1]
150
assert v[0] in ( "'", '"')
151
values.append(eval("str(%s)" % v))
152
return values
153
154
155
156
class SystemType:
157
def __init__(self):
158
self.definition = None
159
self.support = set()
160
161
class general(object):
162
pass
163
164
reservedswitches = (
165
"NO_COMPILE",
166
"TEMPDIR",
167
)
168
169
Spyder = general()
170
171
inheritedtypes = {}
172
def reset():
173
global unimethods, unimethodlist, macros, statements, inheritedtypes, Spyder, corechecksum
174
global converters, concatenators
175
unimethods = {}
176
var = {}
177
unimethodlist = []
178
macros = set()
179
statements = {}
180
converters = []
181
concatenators = []
182
inheritedtypes = {}
183
corechecksum = 0
184
185
186
def defineunimethod(name, pointer):
187
if name in unimethods:
188
raise Exception("system error: duplicate definition of unimethod %s" % name)
189
unimethods[name] = pointer
190
unimethodlist.append(name)
191
def definemacro(pointer):
192
macros.add(pointer)
193
def definestatement(name, pointer):
194
if name in statements:
195
raise Exception("system error: duplicate definition of statement %s" % name)
196
statements[name] = pointer
197
198
def parse(lines, switches, switchlist,spyderdir):
199
switchvars = []
200
currnamespace = ""
201
currswitch = ""
202
spacing = [0,]
203
tabin = False
204
linenr = 0
205
toplevelfiles = []
206
tmpvars = [{}]
207
metavars = {"nocripple":"off"}
208
if spyderdir != None: tmpvars[-1]["spyderdir"] = spyderdir
209
210
while len(lines) > 0:
211
l, lines = lines[0], lines[1:]
212
l = l.replace('\r\n', "")
213
l = l.replace('\n', "")
214
linenr += 1
215
if l.strip().startswith("@"):
216
ll = l.strip()[1:].split("=")
217
if len(ll) == 2:
218
metavars[ll[0].strip()] = ll[1].strip()
219
else:
220
raise Exception("Invalid metavariable statement: " + l)
221
comment = l.find("#")
222
if comment > -1:
223
special = False
224
if metavars["nocripple"] == "on":
225
ll = l.strip()
226
indent = len(l) - len(ll)
227
if ll.startswith("##"):
228
ll = ll[2:].strip()
229
if ll.startswith("NOCRIPPLED"):
230
header = " " * indent
231
if indent == 0:
232
header = "##"
233
else: special = True
234
l = header + ll[len("NOCRIPPLED"):].strip()
235
if special == False:
236
l = l[:comment]
237
if len(l.lstrip()) == 0: continue
238
dollarstart = 0
239
while 1:
240
dollar = l.find("$",dollarstart)
241
if dollar == -1: break
242
for endvar in range(dollar+1,len(l)):
243
if l[endvar].isalnum() == False or l[endvar] == " ":
244
break
245
else: endvar = len(l)
246
varnam = l[dollar+1:endvar]
247
if varnam not in tmpvars[-1]:
248
raise Exception("Unknown variable: '%s'" % varnam)
249
l = l[:dollar] + tmpvars[-1][varnam] + l[endvar:]
250
dollarstart = dollar + len(tmpvars[-1][varnam])
251
currspacing = len(l) - len(l.lstrip())
252
if tabin == True:
253
if currspacing < spacing[-1]:
254
raise Exception("Indentation error in line %d" % linenr)
255
elif currspacing == spacing[-1]:
256
spacing.append(999999)
257
tabout = True
258
else:
259
spacing.append(currspacing)
260
tabin = False
261
tabout = False
262
else: tabout = True
263
264
if tabout == True:
265
if currspacing > spacing[-1]:
266
raise Exception("Indentation error in line %d" % linenr)
267
if currspacing < spacing[-1]:
268
counter = -1
269
for n in range(len(spacing)-2,-1,-1):
270
if currspacing == spacing[n]:
271
counter = len(spacing) - 2 - n
272
break
273
if counter == -1: raise Exception("Indentation error in line %d" % linenr)
274
for n in range(counter+1):
275
if currnamespace != "":
276
switches[currswitch].append("**")
277
currnamespace = ""
278
elif currswitch != "":
279
currswitch = ""
280
if len(tmpvars) > 2:
281
for k in tmpvars[-1]:
282
if k not in tmpvars[-2]: tmpvars[-2][k] = tmpvars[-1][k]
283
tmpvars.pop()
284
spacing.pop()
285
l = l.strip()
286
if l[0] != "*" and l[-1] == ":" :
287
if currnamespace != "": raise Exception("System error: %s:\n Modules may not be nested" % l)
288
currnamespace = l[:-1].rstrip()
289
tmpvars.append(dict(tmpvars[-1]))
290
tabin = True
291
switches[currswitch].append(l)
292
continue
293
if l[0] == "*":
294
ll = l.split()
295
found = False
296
for com in ("*path", "*delpath", "*delfile", "*replacefile", "*include", "*import", "*export"):
297
if ll[0] == com:
298
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
299
found = True
300
for com in ("*enable", "*disable", "*select", "*deselect"):
301
if ll[0] == com:
302
found = True
303
304
if found == False:
305
if ll[0] == "*patch" :
306
if currnamespace != "": raise Exception("System error: %s:\n %s only allowed outside modules" % (l, ll[0]))
307
currnamespace = l[len(ll[0]):].rstrip()
308
if currnamespace[-1] == ":": currnamespace = currnamespace[:-1]
309
tmpvars.append(dict(tmpvars[-1]))
310
tabin = True
311
elif ll[0] == "*switch":
312
if currswitch != "": raise Exception("System error: %s:\switches cannot be nested" % l)
313
if currnamespace != "": raise Exception("System error: %s:\n %s only allowed outside modules" % (l, ll[0]))
314
currswitch = l[len(ll[0]):].lstrip().upper()
315
if currswitch[-1] == ":": currswitch = currswitch[:-1]
316
if currswitch in reservedswitches: raise Exception("System error: switchname %s has been reserved" % currswitch)
317
if currswitch not in switches:
318
switches[currswitch] = []
319
switchlist.append(currswitch)
320
tabin = True
321
continue
322
else:
323
raise Exception("unknown command %s" % l)
324
switches[currswitch].append(l)
325
continue
326
if l.find("=") > -1:
327
varstat = l.split('=')
328
varnam=varstat[0].strip()
329
varval=varstat[1].strip()
330
if len(varnam.split()) > 1:
331
raise Exception("system error: invalid command\n%s" % l)
332
tmpvars[-1][varnam] = varval
333
switchvars.append((currswitch,varnam,varval))
334
continue
335
if len(l.split()) > 1:
336
raise Exception("system error: invalid command\n%s" % l)
337
if currnamespace == "":
338
if l.upper().endswith(".FLY"):
339
try:
340
dat = open(l).readlines()
341
except:
342
raise Exception("Cannot read %s" % l)
343
flydir = os.path.dirname(os.path.abspath(l))
344
lines = dat + lines
345
lines = ["installdir = %s\n" % flydir] + lines
346
else:
347
toplevelfiles.append(l)
348
continue
349
switches[currswitch].append(l)
350
continue
351
for n in range(len(spacing)):
352
if currnamespace != "":
353
switches[currswitch].append("**")
354
currnamespace = ""
355
elif currswitch != "":
356
currswitch = ""
357
return toplevelfiles, switchvars
358
359
def assess_enabling(a, spyderdir, has_command_tempdir = None):
360
global var, has_tempdir, systempdir
361
if spyderdir != None: var["spyderdir"] = spyderdir
362
if has_command_tempdir == None:
363
has_command_tempdir = has_tempdir
364
first = (len(a.checked) == 0)
365
for swvar in a.switchvars:
366
sw,varnam,varval = swvar
367
if sw == a.currswitch:
368
var[varnam] = varval
369
if varnam.lower() == "tempdir":
370
if not has_command_tempdir:
371
systempdir = varval
372
changetempdir(systempdir)
373
has_tempdir = True
374
375
for l in a.switches[a.currswitch]:
376
if l[0] == "*":
377
ll = l.split()
378
if ll[0] == "*enable":
379
ll = l.split()
380
switch = l[len(ll[0]):].lstrip().upper()
381
if switch in a.switches: #No checking if switch is defined!
382
a.enabled.add(switch)
383
if ll[0] == "*disable":
384
ll = l.split()
385
switch = l[len(ll[0]):].lstrip().upper()
386
if switch in a.enabled: #No checking if switch is defined!
387
a.enabled.remove(switch)
388
a.checked.add(a.currswitch)
389
if first:
390
change = True
391
while change == True:
392
change = False
393
for switch in a.switchlist:
394
if switch not in a.checked and switch in a.enabled:
395
change = True
396
a.currswitch = switch
397
assess_enabling(a,spyderdir,has_command_tempdir)
398
399
def assemble(namespaces, lines):
400
currnamespace = ""
401
ignore = False
402
disabled = set()
403
while len(lines) > 0:
404
l = lines[0]
405
lines = lines[1:]
406
l = l.strip()
407
if l[0] != "*" and l[-1] == ":" :
408
if currnamespace != "": raise Exception("System error: %s:\n Modules may not be nested" % l)
409
currnamespace = l[:-1].rstrip()
410
if currnamespace in namespaces:
411
raise Exception("System error: Module %s already defined" % (currnamespace))
412
if currnamespace not in disabled:
413
namespaces[currnamespace] = general()
414
nam = namespaces[currnamespace]
415
nam.order = len(namespaces)
416
nam.name = currnamespace
417
nam.iscore = None
418
nam.paths = []
419
nam.files = []
420
nam.codefiles = []
421
nam.dependent = set()
422
nam.defined = {}
423
nam.supported = set()
424
nam.included = set()
425
nam.precompiled = None
426
nam.compiled = False
427
nam.checksum = None
428
nam.checksum_verified = False
429
nam.source = ""
430
nam.filecompiled = 0
431
nam.id = len(namespaces)
432
nam.imports = []
433
nam.exports = []
434
continue
435
if ignore == True and ll[0] != "**": continue
436
if l[0] == "*":
437
ll = l.split()
438
if ll[0] == "**":
439
currnamespace = ""
440
ignore = False
441
elif ll[0] == "*path":
442
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
443
p = l[len(ll[0]):].lstrip()
444
nam.paths.append(p)
445
elif ll[0] == "*import":
446
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
447
p = l[len(ll[0]):].lstrip()
448
nam.imports.append(p)
449
elif ll[0] == "*export":
450
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
451
p = l[len(ll[0]):].lstrip()
452
nam.exports.append(p)
453
elif ll[0] == "*delpath":
454
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
455
p = l[len(ll[0]):].lstrip()
456
if p in nam.paths:
457
nam.paths.remove(p)
458
elif ll[0] == "*delfile":
459
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
460
p = l[len(ll[0]):].lstrip()
461
if p in nam.files:
462
nam.files.remove(p)
463
elif ll[0] == "*replacefile":
464
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
465
p = l[len(ll[0]):].lstrip()
466
pp = p.split()
467
if len(pp) != 2:
468
raise Exception("System error: Malformed *replacefile: %s" % (l))
469
pp[0] = pp[0].strip()
470
pp[1] = pp[1].strip()
471
if pp[0] in nam.files:
472
nam.files[nam.files.index(pp[0])] = pp[1]
473
elif ll[0] == "*include":
474
if currnamespace == "": raise Exception("System error: %s:\n %s only allowed in modules" % (l, ll[0]))
475
nam.included.add(ll[1])
476
elif ll[0] == "*patch" :
477
if currnamespace != "": raise Exception("System error: %s:\n %s only allowed outside modules" % (l, ll[0]))
478
currnamespace = l[len(ll[0]):].lstrip()
479
if currnamespace[-1] == ":": currnamespace = currnamespace[:-1]
480
if currnamespace not in namespaces:
481
if currnamespace in disabled:
482
ignore = True
483
else:
484
raise Exception("System error: Module %s cannot be patched since it is undefined" % (currnamespace))
485
nam = namespaces[currnamespace]
486
elif ll[0] == "*select": #modules that are defined later are still added!
487
delspaces = []
488
for nam in namespaces:
489
if nam not in ll[1:]:
490
disabled.add(nam)
491
delspaces.append(nam)
492
for nam in delspaces: namespaces.pop(nam)
493
elif ll[0] == "*deselect":
494
delspaces = []
495
for nam in namespaces:
496
if nam in ll[1:]:
497
disabled.add(nam)
498
delspaces.append(nam)
499
for nam in delspaces: namespaces.pop(nam)
500
elif ll[0] == "*enable" :
501
continue
502
elif ll[0] == "*disable" :
503
continue
504
else:
505
raise Exception("unknown command %s" % l)
506
continue
507
if currnamespace == "": raise Exception
508
psplit = l.split(".")
509
if len(psplit) != 2: raise Exception("System error: file name %s should contain exactly one dot" % l)
510
ext = psplit[1].upper()
511
if ext == "SPY" or ext == "PY":
512
nam.files.append(l)
513
elif ext == "FLY":
514
if not os.path.exists(l):
515
raise Exception("System error: file name %s does not exist" % l)
516
lines = open(l).readlines() + lines
517
else:
518
raise Exception("System error: file name %s must be Spyder or Python file" % l)
519
520
521
522
def analyze_python_file(system, fil, lines, namespace, iscompiled, make_definitions=True):
523
for l in lines:
524
l = l.strip()
525
if len(l) < 2: continue
526
if l[0] != "#": break
527
if l[:2] != "##": continue
528
l = l[2:].strip()
529
l1 = l.split()[0].rstrip()
530
l2 = l[len(l1):].lstrip()
531
l2 = l2.split(",")
532
for n in range(0,len(l2)): l2[n] = l2[n].strip()
533
if l1 == "DEFINE":
534
if namespace.iscore == True:
535
raise Exception("System error in %s: module may not contain core files and DEFINE metastatements" % (namespace.name))
536
namespace.iscore = False
537
for ll in l2:
538
if not validvar(ll):
539
raise Exception("System error in file %s: defined type %s must start with a capital" % (fil,ll))
540
if ll == "Spyder" or ll == "Delete": raise Exception("System error in file %s: defined type may not be 'Spyder'" % (fil))
541
if ll in system and system[ll].definition != None and system[ll].definition != namespace.name:
542
raise Exception("System error in file %s: type %s already defined in %s" % (fil, ll, system[ll].definition))
543
if ll not in system: system[ll] = SystemType()
544
if make_definitions == True: system[ll].definition = namespace.name
545
namespace.defined[ll] = ""
546
elif l1 == "SUPPORT":
547
if namespace.iscore == True:
548
raise Exception("System error in %s: module may not contain core files and SUPPORT metastatements" % (namespace.name))
549
namespace.iscore = False
550
for ll in l2:
551
if ll == "None": continue
552
if not validvar(ll):
553
raise Exception("System error in file %s: supported type %s must start with a capital" % (fil,ll))
554
if ll == "Spyder" or ll == "Delete": raise Exception("System error in file %s: defined type may not be 'Spyder' or 'Delete'" % (fil))
555
if ll not in system: system[ll] = SystemType()
556
system[ll].support.add(namespace.name)
557
namespace.supported.add(ll)
558
elif l1 == "CHECKSUM":
559
if iscompiled == False:
560
raise Exception("System error in file %s: only compiled file may contain CHECKSUM metastatement" % (fil))
561
if len(l2) < 1:
562
raise Exception("System error in file %s: malformed CHECKSUM metastatement" % (fil))
563
checksum = l[len(l1):].lstrip()
564
namespace.checksum = checksum #checksum of the SOURCE plus CORE SOURCE, not of the file itself!
565
elif l1 == "NEEDED":
566
if namespace.iscore == True:
567
raise Exception("System error in %s: module may not contain core files and NEEDED metastatements" % (namespace.name))
568
namespace.iscore = False
569
for ll in l2:
570
if not validvar(ll):
571
raise Exception("System error in file %s: needed type %s must start with a capital" % (fil,ll))
572
if ll == "Spyder" or ll == "Delete": raise Exception("System error in file %s: needed type may not be 'Spyder'" % (fil))
573
namespace.neededtypes.add(ll)
574
elif l1 == "DEPENDENT":
575
if iscompiled == False:
576
raise Exception("System error in file %s: only compiled file may contain NEEDED metastatement" % (fil))
577
if namespace.iscore == True:
578
raise Exception("System error in %s: module may not contain core files and DEPENDENT metastatements" % (namespace.name))
579
namespace.iscore = False
580
l2 = l[len(l1):].split()
581
if len(l2) != 2:
582
raise Exception("System error in file %s: malformed DEPENDENT metastatement" % (fil))
583
checksum = l2[1].strip()
584
namespace.dependent.add((l2[0], checksum))
585
elif l1 == "CORE":
586
if namespace.iscore == False:
587
raise Exception("System error in %s: module may not contain core files and Spyder files/meta statements" % (namespace.name))
588
namespace.iscore = True
589
else: raise Exception("System error in file %s:\n Unrecognized metacommand %s" % (fil, l1))
590
if iscompiled == True and namespace.iscore != True: namespace.iscore = False
591
592
anyfound = False
593
def compile_core_namespace(system, namespace):
594
global corechecksum, anyfound
595
source = ""
596
path = ""
597
if len(namespace.paths) > 0: path = namespace.paths[0]
598
if has_tempdir: fil = namespace.name
599
else: fil = os.path.join(path, namespace.name)
600
fil += "_MODULE_spy.py"
601
namespace.compiledfile = fil
602
namespace.precompiled = False
603
namespace.neededtypes = set()
604
if os.path.exists(fil):
605
f = open(fil)
606
d = f.read().replace('\r\n','\n').rstrip('\n')
607
lines = d.split("\n")
608
namespace.source = d
609
f.close()
610
analyze_python_file(system, fil, lines, namespace, True, False)
611
namespace.precompiled = None
612
613
for filename in namespace.files:
614
ok = False
615
for p in namespace.paths:
616
fil = os.path.join(p, filename)
617
if os.path.exists(fil):
618
f = open(fil)
619
d = f.read().replace('\r\n','\n').rstrip('\n')
620
lines = d.split('\n')
621
source += d + '\n'
622
namespace.codefiles.append(fil)
623
f.close()
624
ok = True
625
if filename.upper().endswith("SPY"):
626
if namespace.iscore == True:
627
raise Exception("System error in %s: module may not contain core files and Spyder files" % (namespace.name))
628
namespace.iscore = False
629
elif filename.upper().endswith("PY"):
630
if namespace.precompiled == False:
631
analyze_python_file(system, fil,lines, namespace, False)
632
else:
633
raise Exception("Only .spy and .py files can be present in modules")
634
break
635
if ok == False:
636
if anyfound == False:
637
raise Exception("Spyder could not locate its core functions. Your Spyder install seems to be broken")
638
else: raise Exception("System error: cannot find %s in module %s" % (filename,namespace.name))
639
anyfound = True
640
if namespace.iscore == True:
641
checksum = hex(zlib.adler32(source))
642
if namespace.precompiled == None:
643
namespace.precompiled = True
644
if checksum != namespace.checksum:
645
namespace.precompiled = False
646
corechecksum = zlib.adler32(source, corechecksum)
647
if namespace.precompiled == False:
648
source = ""
649
#print >> sys.stderr, "compiling core module '%s'..." % namespace.name
650
f = open(namespace.compiledfile, "w")
651
f.write("## CORE\n")
652
f.write("## CHECKSUM %s\n\n" % checksum)
653
for fil in namespace.codefiles:
654
l = "execfile(r\"%s\")\n" % os.path.abspath(fil)
655
source += l
656
f.write(l)
657
658
f.close()
659
660
filelist = namespace.compiledfile[:-3] + "_file.list"
661
f = open(filelist, "w")
662
for l in namespace.codefiles:
663
f.write(l + '\n')
664
f.close()
665
else:
666
namespace.codefiles = []
667
f = open(namespace.compiledfile[:-3] + "_file.list")
668
d = f.readlines()
669
for l in d:
670
ll = l.rstrip('\n')
671
namespace.codefiles.append(ll)
672
f.close()
673
674
675
#print >> sys.stderr, "executing core module '%s'..." % namespace.name
676
shell = {'Spyder':Spyder}
677
Spyder.__currpaths__ = namespace.paths
678
head, tail = headtailcode(namespace.name, namespace.paths, namespace.imports,namespace.exports)
679
#source = head + '\n' + source + tail + '\n'
680
try:
681
exec(head % dict(shell="shell"))
682
exec(source, shell)
683
exec(tail % dict(shell="shell"))
684
except:
685
#dump = open("dump.py", "w")
686
#dump.write(source)
687
#dump.close()
688
raise
689
for var in shell:
690
if validvar(var) and var != 'Spyder':
691
inheritedtypes[var] = {}
692
namespace.source = source
693
namespace.compiled = True
694
namespace.calledtypes = set()
695
else: namespace.defined = {}
696
697
def compile_namespace(system, namespaces, namespace):
698
if namespace.checksum_verified == False:
699
compiledfiles = set()
700
fil = namespace.compiledfile
701
if not (os.path.exists(fil[:-3] + "_called.types") and os.path.exists(fil[:-3] + "_defined.types") and os.path.exists(fil[:-3] + "_file.list")):
702
namespace.precompiled = False
703
namespace.supported = set()
704
namespace.defined = {}
705
namespace.source = ""
706
namespace.dependent = set()
707
else:
708
f = open(fil[:-3] + "_file.list")
709
d = f.readlines()
710
f.close()
711
for l in d:
712
if l.strip().endswith("_spy.py") and not os.path.exists(l.strip()):
713
namespace.precompiled = False
714
namespace.supported = set()
715
namespace.defined = {}
716
namespace.source = ""
717
namespace.dependent = set()
718
break
719
else:
720
for l in d:
721
ll = l.rstrip('\n')
722
compiledfiles.add(ll)
723
foundfiles = set()
724
source = ""
725
for n in range(0,len(namespace.files)):
726
filename = namespace.files[n]
727
ok = False
728
for p in namespace.paths:
729
fil = os.path.join(p, filename)
730
if os.path.exists(fil):
731
ok = True
732
if filename.upper().endswith("SPY"):
733
if has_tempdir:
734
mdir = namespace.name
735
if not os.path.exists(mdir):
736
os.mkdir(mdir)
737
foundfiles.add(mdir + os.sep + os.path.basename(fil[:-4])+"_spy.py")
738
else:
739
foundfiles.add(fil[:-4]+"_spy.py")
740
f = open(fil)
741
d = f.read().replace('\r\n','\n').rstrip('\n')
742
source += d + '\n\n'
743
f.close()
744
else:
745
foundfiles.add(fil)
746
f = open(fil)
747
d = f.read().replace('\r\n','\n').rstrip('\n')
748
source += d + '\n\n'
749
lines = d.split('\n')
750
f.close()
751
namespace.files[n] = fil
752
break
753
if ok == False: raise Exception("System error: cannot find %s" % filename)
754
checksum = hex(zlib.adler32(source, corechecksum))
755
if namespace.precompiled == None:
756
if checksum != namespace.checksum or len(foundfiles ^ compiledfiles) > 0:
757
namespace.precompiled = False
758
namespace.supported = set()
759
namespace.defined = {}
760
namespace.source = ""
761
namespace.dependent = set()
762
else: namespace.precompiled = True
763
namespace.checksum = checksum
764
namespace.checksum_verified = True
765
766
if namespace.precompiled == True:
767
if not hasattr(namespace,"dependent_copy"):
768
namespace.dependent_copy = namespace.dependent
769
#check if the dependency namespaces are in the system, and identical in checksum
770
dep = list(namespace.dependent_copy)
771
namespace.dependent_copy = set()
772
for n in range(0, len(dep)):
773
d = dep[n]
774
ok = True
775
if d[0] not in namespaces:
776
ok = False
777
elif namespaces[d[0]].compiled == False:
778
namespace.dependent_copy.add(d)
779
elif namespaces[d[0]].checksum != d[1]:
780
ok = False
781
if ok == False:
782
namespace.precompiled = False
783
namespace.supported = set()
784
namespace.defined = {}
785
namespace.neededtypes = set()
786
namespace.dependent = set()
787
namespace.source = ""
788
break
789
if ok == True: namespace.dependent = namespace.dependent_copy
790
if namespace.precompiled == True and len(namespace.dependent) == 0: #every test has been passed
791
analyze_python_file(system, namespace.compiledfile, namespace.source.splitlines(), namespace, True)
792
namespace.compiled = True
793
794
f = open(namespace.compiledfile[:-3] + "_defined.types")
795
d = f.read().replace('\r\n','\n')
796
f.close()
797
for l in d.splitlines():
798
sep = l.index(" ")
799
typ, s = l[:sep], l[sep+1:]
800
namespace.defined[typ] = {}
801
mask = ""
802
pos = 0
803
p = quotematch.finditer(s)
804
for pp in p:
805
mask += s[pos:pp.start()] + "*" * (pp.end()-pp.start())
806
pos = pp.end()
807
mask += s[pos:]
808
mask = mask.replace('{', '[').replace('}',']')
809
mask = mask.replace('(', '[').replace(')',']')
810
mask = mask.replace(r'\\', '**')
811
m = mask
812
bm = m.find('[')
813
if bm > -1:
814
bm2 = m.rfind(']')
815
cm = m.find(',')
816
s = s[bm+1:bm2]
817
m = m[bm+1:bm2]
818
namespace.defined[typ] = dictparse(s,m)
819
f = open(namespace.compiledfile[:-3] + "_called.types")
820
d = f.read().replace('\r\n','\n')
821
f.close()
822
namespace.calledtypes = set(d.splitlines())
823
824
namespace.codefiles = []
825
f = open(namespace.compiledfile[:-3] + "_file.list")
826
d = f.readlines()
827
for l in d:
828
ll = l.rstrip('\n')
829
namespace.codefiles.append(ll)
830
f.close()
831
for var in namespace.defined:
832
if var not in system:
833
system[var] = SystemType()
834
if system[var].definition != namespace.name and system[var].definition != None:
835
raise Exception("compile error: type %s twice defined in modules %s and %s" % (var, namespace.name, system[var].definition))
836
system[var].definition = namespace.name
837
if namespace.precompiled == False:
838
#sys.exit()
839
change = True
840
#filedefined = {}
841
while change:
842
change = False
843
newnt = copy(namespace.neededtypes)
844
for nt in namespace.neededtypes:
845
if nt not in system:
846
system[nt] = SystemType()
847
if system[nt].definition != None:
848
depnam = namespaces[system[nt].definition]
849
if depnam.id > namespace.id:
850
raise Exception("ERROR: Module %s defines type %s, which is dependent on module %s, which is defined later in system.fly\nChange the order of modules" %(namespace.name, nt, depnam.name))
851
if depnam.compiled == True:
852
namespace.dependent.add((depnam.name, depnam.checksum))
853
newnt.remove(nt)
854
elif depnam.name == namespace.name:
855
newnt.remove(nt)
856
namespace.neededtypes = newnt
857
if len(namespace.files) == 0:
858
namespace.calledtypes = set()
859
namespace.compiled = True
860
elif len(namespace.neededtypes) == 0:
861
filename = namespace.files[namespace.filecompiled]
862
f = open(filename)
863
s = f.read().replace('\r\n','\n')
864
f.close()
865
if filename.upper().endswith(".SPY"):
866
#print "NAMESPACE", namespace.name
867
#print "FILE", filename
868
inheritedtypes0 = inheritedtypes.copy()
869
for v in namespace.defined:
870
inheritedtypes0[v] = namespace.defined[v]
871
result, neededtypes, definedtypes, supportedtypes = Spydercompile(Spyder, s, unimethodlist, unimethods, macros, statements, inheritedtypes0, namespace.name, filename)
872
#print "RESULT", result
873
#print "NEEDEDTYPES", neededtypes
874
#print "DEFINEDTYPES", definedtypes
875
#print "SUPPORTEDTYPES", supportedtypes
876
if result == None:
877
namespace.neededtypes = neededtypes
878
else:
879
result = result.lstrip('\n') + '\n'
880
linenr = len(result.splitlines())
881
if has_tempdir:
882
mdir = namespace.name
883
if not os.path.exists(mdir):
884
os.mkdir(mdir)
885
spynam = mdir + os.sep + os.path.basename(filename)[:-4]+"_spy.py"
886
else: spynam = filename[:-4]+"_spy.py"
887
spyfile = open(spynam, "w")
888
spyfile.write(result)
889
spyfile.close()
890
namespace.codefiles[namespace.filecompiled] = spynam
891
namespace.source += result
892
namespace.supported = namespace.supported.union(supportedtypes)
893
for var in definedtypes:
894
if var in namespace.defined:
895
raise Exception("compile error: type %s twice defined in module %s" % (var, namespace.name))
896
if var not in system:
897
system[var] = SystemType()
898
if system[var].definition != namespace.name and system[var].definition != None:
899
raise Exception("compile error: type %s twice defined in modules %s and %s" % (var, namespace.name, system[var].definition))
900
system[var].definition = namespace.name
901
namespace.defined[var] = definedtypes[var]
902
change = True
903
namespace.filecompiled += 1
904
else:
905
#if filename not in filedefined:
906
s = s.lstrip('\n') + '\n\n'
907
namespace.source += s
908
lines = s.split("\n")
909
tmpnamespace = general()
910
tmpnamespace.name = namespace.name
911
tmpnamespace.defined = {}
912
tmpnamespace.neededtypes = set()
913
tmpnamespace.supported = set()
914
tmpnamespace.iscore = False
915
analyze_python_file(system, filename,lines, tmpnamespace, False)
916
#filedefined[filename] = tmpnamespace.defined
917
namespace.supported = namespace.supported.union(tmpnamespace.supported)
918
if len(tmpnamespace.neededtypes) > 0:
919
namespace.neededtypes = tmpnamespace.neededtypes
920
#change = True
921
#continue
922
#if filedefined[filename] > 0:
923
#if filename.upper().endswith(".SPY"):
924
#shell = {'Spyder':Spyder}
925
#dump = open("dump.py", "w")
926
#dump.write(namespace.source)
927
#dump.close()
928
#exec(namespace.source, shell)
929
#ok = True
930
for var in tmpnamespace.defined:
931
if var in namespace.defined:
932
raise Exception("compile error: type %s twice defined in module %s" % (var, namespace.name))
933
namespace.defined[var] = {}
934
if var not in system:
935
system[var] = SystemType()
936
if system[var].definition != namespace.name and system[var].definition != None:
937
raise Exception("compile error: type %s twice defined in modules %s and %s" % (var, namespace.name, system[var].definition))
938
system[var].definition = namespace.name
939
#namespace.defined[var] = definedtypes[var]
940
namespace.defined[var] = tmpnamespace.defined[var]
941
942
943
change = True
944
namespace.filecompiled += 1
945
if namespace.filecompiled == len(namespace.files) and len(namespace.neededtypes) == 0:
946
if "SILENT" not in assesspar.enabled: print >> sys.stderr, "compiling module '%s'" % namespace.name
947
#filteredsource0 = commentmatch.split(namespace.source)
948
#filteredsource = ""
949
#for f in filteredsource0: filteredsource += f
950
filteredsource = namespace.source
951
f = open(namespace.compiledfile, "w")
952
f.write("\n## CHECKSUM %s\n\n" % namespace.checksum)
953
for v in namespace.defined:
954
f.write("## DEFINE %s\n" % v)
955
for v in namespace.supported:
956
if v not in namespace.defined:
957
f.write("## SUPPORT %s\n" % v)
958
for v in namespace.dependent:
959
f.write("## DEPENDENT %s %s\n" % (v[0], v[1]))
960
f.write("\n" + filteredsource)
961
f.close()
962
definedtypes = namespace.compiledfile[:-3] + "_defined.types"
963
f = open(definedtypes, "w")
964
#f.write(repr(namespace.defined))
965
for t in namespace.defined:
966
print >>f, t, repr(namespace.defined[t])
967
f.close()
968
969
filelist = namespace.compiledfile[:-3] + "_file.list"
970
f = open(filelist, "w")
971
for l in namespace.codefiles:
972
print >> f, l
973
f.close()
974
namespace.calledtypes = get_called_types(namespace.source)
975
calledtypes = namespace.compiledfile[:-3] + "_called.types"
976
f = open(calledtypes, "w")
977
for t in namespace.calledtypes:
978
print >> f, t
979
f.close()
980
981
namespace.compiled = True
982
change = False
983
984
if namespace.compiled == True:
985
for v in namespace.defined:
986
inheritedtypes[v] = namespace.defined[v]
987
Spyder.core.__defined[v] = namespace.defined[v]
988
989
def visit_node(node, ret):
990
nodetype = node.__class__.__name__
991
if nodetype == "Name":
992
name = node.getChildren()[0]
993
name0 = name
994
while name0.endswith("Array"): name0 = name0[:-len("Array")]
995
if validvar(name0):
996
ret.add(name)
997
for child in node.getChildNodes():
998
visit_node(child, ret)
999
1000
def get_called_types(code, raise_syntaxerror = True):
1001
try:
1002
try:
1003
ast = compiler.parse(code)
1004
except SyntaxError:
1005
#at any point, it will crash, just depends if we want to let it crash now or later...
1006
try:
1007
exec(code)
1008
except SyntaxError:
1009
if raise_syntaxerror == False: return set()
1010
raise
1011
except:
1012
raise
1013
except:
1014
#print "Compilation error, check dump.py to fix it"
1015
#dump = open("dump.py", "w")
1016
#dump.write(code)
1017
#dump.close()
1018
raise
1019
ret = set()
1020
visit_node(ast, ret)
1021
return ret
1022
1023
def determine_called_namespaces(system, namespaces, name):
1024
namespace = namespaces[name]
1025
if hasattr(namespace, "called"): return
1026
namespace.called = []
1027
for t in namespace.calledtypes:
1028
while t.endswith("Array"): t = t[:-len("Array")]
1029
if t not in system or system[t].definition not in namespaces:
1030
raise Exception("System error: module %s needs Spyder type %s and no system module can provide it" % (name,t))
1031
tt = system[t]
1032
if namespaces[tt.definition].id > namespace.id:
1033
raise Exception("System error: module %s needs Spyder type %s and the defining system module %s is later in system.fly\nPlease change the order of modules" % (name,t,tt.definition))
1034
newnames = set((tt.definition,))
1035
if hasattr(tt, "supported") and tt.supported != None: newnames = newnames.union(set(tt.supported))
1036
for n in newnames:
1037
if n != name:
1038
determine_called_namespaces(system, namespaces, n)
1039
if n not in namespace.called:
1040
for nn in namespaces[n].called:
1041
if nn not in namespace.called: namespace.called.append(nn)
1042
namespace.called.append(name)
1043
for i in namespace.included:
1044
if i not in namespace.called: namespace.called.append(i)
1045
1046
def headtailcode(modulename, paths, imports, exports):
1047
ret1 = "Spyder.core.__currpaths = %s\n" % repr(paths)
1048
ret1 += "if \"%s\" not in Spyder.__dict__: Spyder.%s = Spyder.core.namespace()\n" % (modulename, modulename)
1049
for i in imports:
1050
txt = "\""+ i + "\",\"" + i + "\""
1051
ii = i.split()
1052
1053
nam = ii[0]
1054
val = ii[0]
1055
if (len(ii) != 1 and (len(ii) != 3 or ii[1] != "as")) or validvar(ii[0]):
1056
raise Exception("Malformed import statement in %s: *import %s" % (modulename,i))
1057
if len(ii) == 3:
1058
txt = "\"" + ii[0] + "\",\"" + ii[2] + "\""
1059
val = ii[2]
1060
if val.find(".") > -1:
1061
val = val[:val.index(".")]
1062
ret1 += "Spyder.core_init.__fullimport(%s, %%(shell)s)\n" % (txt)
1063
ret1 += "Spyder.%s.%s = %%(shell)s[\"%s\"]\n" % (modulename, val, val)
1064
ret2 = ""
1065
for i in exports:
1066
ii = i.split()
1067
nam = ii[0]
1068
val = ii[0]
1069
if (len(ii) != 1 and (len(ii) != 3 or ii[1] != "as")) or validvar(ii[0]):
1070
raise Exception("Malformed export statement in %s: *export %s" % (modulename,i))
1071
if len(ii) == 3:
1072
val = ii[2]
1073
if nam.find(".") > -1:
1074
raise Exception("Malformed export statement in %s: *export %s" % (modulename,i))
1075
ret2 += "Spyder.%s.%s = %%(shell)s[\"%s\"]\n" % (modulename, val, nam)
1076
return ret1,ret2
1077
1078
def import_namespace(modulename,files, paths,imports,exports,deftypes):
1079
ret = ""
1080
head, tail = headtailcode(modulename, paths, imports, exports)
1081
ret = "__subshell = globals().copy()\n"
1082
ret += head % dict(shell="__subshell") + "\n"
1083
for f in files:
1084
ret += "execfile(r\"%s\", __subshell)\n" % os.path.abspath(f)
1085
for typ in deftypes:
1086
ret += "%s = __subshell[\"%s\"]\n" % (typ, typ)
1087
for n in range(Spyder.core.__arraycount[typ]):
1088
typ2 = typ + (n+1) * "Array"
1089
ret += "%s = __subshell[\"%s\"]\n" % (typ2, typ2)
1090
ret += tail % dict(shell="__subshell") + "\n"
1091
return ret
1092
1093
def systemcompile(lines, enabledswitches=set(),spyderdir=None,has_tempdir0=False):
1094
global has_tempdir, systempdir, namespaces, namespacelist, assesspar
1095
systempdir = None
1096
has_tempdir = has_tempdir0
1097
system = {}
1098
namespaces = {}
1099
switches = {"":[]}
1100
switchlist = ["",]
1101
toplevelfiles, switchvars = parse(lines, switches, switchlist,spyderdir)
1102
assesspar = general()
1103
assesspar.switches = switches
1104
assesspar.switchvars = switchvars
1105
assesspar.switchlist = switchlist
1106
assesspar.currswitch = ""
1107
assesspar.enabled = enabledswitches
1108
assesspar.checked = set()
1109
assess_enabling(assesspar, spyderdir)
1110
l = []
1111
l += switches[""]
1112
for s in switchlist:
1113
if s in assesspar.enabled: l += switches[s]
1114
for s in enabledswitches:
1115
if s not in switchlist and s != "NOCOMPILE" and s != "SILENT":
1116
print >> sys.stderr, "WARNING: unknown switch %s" % s
1117
assemble(namespaces, l)
1118
namespacelist = []
1119
for nr in range(len(namespaces)):
1120
minn = None
1121
minnr = 999999
1122
for n in namespaces:
1123
if n not in namespacelist and namespaces[n].order < minnr:
1124
minnr = namespaces[n].order
1125
minn = n
1126
namespacelist.append(minn)
1127
#assert len(namespacelist) == len(namespaces)
1128
for n in namespacelist:
1129
compile_core_namespace(system, namespaces[n])
1130
change = True
1131
while change:
1132
change = False
1133
for n in namespacelist:
1134
if namespaces[n].compiled == False:
1135
compile_namespace(system, namespaces, namespaces[n])
1136
if namespaces[n].compiled == True:
1137
change = True
1138
for n in namespacelist:
1139
if namespaces[n].compiled == False:
1140
raise Exception("System error: cannot compile module %s due to the following undefined types:" % namespaces[n].name, list(namespaces[n].neededtypes))
1141
corecode = ""
1142
for n in namespacelist:
1143
if namespaces[n].iscore and len(namespaces[n].codefiles) > 0:
1144
corecode += import_namespace(n, namespaces[n].codefiles, namespaces[n].paths, namespaces[n].imports, namespaces[n].exports, namespaces[n].defined) % dict(shell="currnamespace")
1145
continue
1146
determine_called_namespaces(system, namespaces, n)
1147
1148
type_to_namespaces = {}
1149
for t in system:
1150
n = []
1151
if system[t].definition == None: directspaces = []
1152
else: directspaces = [system[t].definition,]
1153
if hasattr(system[t], "support"):
1154
directspaces += system[t].support
1155
for space in directspaces:
1156
for nam in namespaces[space].called:
1157
nr = namespacelist.index(nam)
1158
if (nam, nr) not in n:
1159
n.append((nam, nr))
1160
type_to_namespaces[t] = n
1161
return corecode, type_to_namespaces, namespaces, toplevelfiles, systempdir
1162
1163
1164
#TO DO:
1165
#- EXECORE and METACORE (METACORE to exec during compilation (Python),
1166
# EXECORE to exec during runtime (target language)
1167
# std CORE to do both (Python)
1168
# this is a FILE attribute not a MODULE attribute
1169
#- SWITCH to assume that the system did not change (no recompile)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1