2
# Twisted, the Framework of Your Internet
3
# Copyright (C) 2001 Matthew W. Lefkowitz
5
# This library is free software; you can redistribute it and/or
6
# modify it under the terms of version 2.1 of the GNU Lesser General Public
7
# License as published by the Free Software Foundation.
9
# This library is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
# Lesser General Public License for more details.
14
# You should have received a copy of the GNU Lesser General Public
15
# License along with this library; if not, write to the Free Software
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
"""S-expression-based persistence of python objects.
20
I do something very much like Pickle; however, pickle's main goal seems to be
21
efficiency (both in space and time); jelly's main goals are security, human
22
readability, and portability to other environments.
25
This is how Jelly converts various objects to s-expressions:
29
List: [1, 2] --> ['list', 1, 2]
31
String: "hello" --> "hello"
35
Dictionary: {'a' : 1, 'b' : 'c'} --> ['dictionary', ['b', 'c'], ['a', 1]]
37
Module: UserString --> ['module', 'UserString']
39
Class: UserString.UserString --> ['class', ['module', 'UserString'], 'UserString']
41
Function: string.join --> ['function', 'join', ['module', 'string']]
43
Instance: s is an instance of UserString.UserString, with a __dict__ {'data': 'hello'}:
44
['instance', ['class', ['module', 'UserString'], 'UserString'], ['dictionary', ['data', 'hello']]]
46
Class Method: UserString.UserString.center:
47
['method', 'center', ['None'], ['class', ['module', 'UserString'], 'UserString']]
49
Instance Method: s.center, where s is an instance of UserString.UserString:
50
['method', 'center', ['instance', ['reference', 1, ['class', ['module', 'UserString'], 'UserString']], ['dictionary', ['data', 'd']]], ['dereference', 1]]
61
from new import instance
62
from new import instancemethod
64
from org.python.core import PyMethod
65
instancemethod = PyMethod
67
None_atom = "None" # N
69
class_atom = "class" # c
70
module_atom = "module" # m
71
function_atom = "function" # f
74
dereference_atom = 'dereference' # D
75
persistent_atom = 'persistent' # p
76
reference_atom = 'reference' # r
79
dictionary_atom = "dictionary" # d
80
list_atom = 'list' # l
82
# immutable collections
83
# (assignment to __dict__ and __class__ still might go away!)
84
tuple_atom = "tuple" # t
85
instance_atom = 'instance' # i
89
unpersistable_atom = "unpersistable"# u
92
types.StringType: "string",
94
types.LongType: "long",
95
types.FloatType: "float",
96
types.ClassType: "class",
97
types.DictType: "dictionary",
98
types.ListType: "list",
99
types.TupleType: "tuple",
100
types.BuiltinMethodType: "builtin_function_or_method",
101
types.FunctionType: "function",
102
types.ModuleType: "module",
103
types.InstanceType: "instance",
104
types.MethodType: "instance_method",
105
types.NoneType: "None",
108
if hasattr(types, 'UnicodeType'):
109
typeNames[types.UnicodeType] = 'unicode'
112
from org.python.core import PyStringMap
113
typeNames[PyStringMap] = "dictionary"
120
This is an instance of a class that comes back when something couldn't be
123
def __init__(self, reason):
125
Initialize an unpersistable object with a descriptive `reason' string.
130
return "Unpersistable(%s)" % repr(self.reason)
133
"""(Internal) This class manages state for a call to jelly()
135
def __init__(self, taster, persistentStore):
139
# `preseved' is a dict of previously seen instances.
141
# `cooked' is a dict of previously backreferenced instances to their `ref' lists.
145
self.persistentStore = persistentStore
147
def _cook(self, object):
149
backreference an object
151
aList = self.preserved[id(object)]
152
newList = copy.copy(aList)
153
# make a new reference ID
155
self._ref_id = self._ref_id + 1
156
# replace the old list in-place, so that we don't have to track the
157
# previous reference to it.
158
aList[:] = [reference_atom, refid, newList]
159
self.cooked[id(object)] = [dereference_atom, refid]
162
def _prepare(self, object):
164
create a list for persisting an object to. this will allow
165
backreferences to be made internal to the object. (circular
168
# create a placeholder list to be preserved
169
self.preserved[id(object)] = []
170
# keep a reference to this object around, so it doesn't disappear!
171
# (This isn't always necessary, but for cases where the objects are
172
# dynamically generated by __getstate__ or getStateToCopyFor calls, it
173
# is; id() will return the same value for a different object if it gets
174
# garbage collected. This may be optimized later.)
175
self.cooker[id(object)] = object
178
def _preserve(self, object, sexp):
180
mark an object's persistent list for later referral
182
#if I've been cooked in the meanwhile,
183
if self.cooked.has_key(id(object)):
184
# replace the placeholder empty list with the real one
185
self.preserved[id(object)][2] = sexp
186
# but give this one back.
187
sexp = self.preserved[id(object)]
189
self.preserved[id(object)] = sexp
192
def jelly(self, object):
193
"""(internal) make a list
195
# if it's been previously backreferenced, then we're done
196
if self.cooked.has_key(id(object)):
197
return self.cooked[id(object)]
198
# if it's been previously seen but NOT backreferenced,
199
# now's the time to do it.
200
if self.preserved.has_key(id(object)):
202
return self.cooked[id(object)]
203
typnm = string.replace(typeNames[type(object)], ' ', '_')
204
### this next block not _necessarily_ correct due to some tricks in
205
### NetJellier's _jelly_instance...
206
## if not self.taster.isTypeAllowed(typnm):
207
## raise InsecureJelly("Type %s not jellyable." % typnm)
208
typfn = getattr(self, "_jelly_%s" % typnm, None)
212
return self.unpersistable("type: %s" % repr(type(object)))
214
def _jelly_string(self, st):
215
"""(internal) Return the serialized representation of a string.
217
This just happens to be the string itself.
221
def _jelly_unicode(self, u):
222
"""(internal) (python2.1 only) UTF-8 encode a unicode string.
224
return ['unicode', u.encode('UTF-8')]
226
def _jelly_int(self, nt):
228
Return the serialized representation of an integer (which is the
233
def _jelly_long(self, ng):
236
Return the serialized representation of a long integer (this will only
237
work for long ints that can fit into a regular integer, currently, but
238
that limitation is temporary)
242
def _jelly_float(self, loat):
244
Return the serialized representation of a float (which is the float
249
### these have to have unjelly equivalents
251
def _jelly_instance(self, instance):
252
'''Jelly an instance.
254
In the default case, this returns a list of 3 items::
256
(instance (class ...) (dictionary ("attrib" "val")) )
258
However, if I was created with a persistentStore method, then that
259
method will be called with the 'instance' argument. If that method
260
returns a string, I will return::
264
# like pickle's persistent_id
265
sxp = self._prepare(instance)
267
if self.persistentStore:
268
persistent = self.persistentStore(instance, self)
269
if persistent is not None:
270
sxp.append(persistent_atom)
271
sxp.append(persistent)
272
elif self.taster.isModuleAllowed(instance.__class__.__module__):
273
if self.taster.isClassAllowed(instance.__class__):
274
sxp.append(instance_atom)
275
sxp.append(self.jelly(instance.__class__))
276
if hasattr(instance, '__getstate__'):
277
state = instance.__getstate__()
279
state = instance.__dict__
280
sxp.append(self.jelly(state))
282
self.unpersistable("instance of class %s deemed insecure" % str(instance.__class__), sxp)
284
self.unpersistable("instance from module %s deemed insecure" % str(instance.__class__.__module__), sxp)
285
return self._preserve(instance, sxp)
288
def _jelly_class(self, klaus):
289
''' (internal) Jelly a class.
290
returns a list of 3 items: (class "module" "name")
292
if self.taster.isModuleAllowed(klaus.__module__):
293
if self.taster.isClassAllowed(klaus):
294
jklaus = self._prepare(klaus)
295
jklaus.append(class_atom)
296
jklaus.append(self.jelly(sys.modules[klaus.__module__]))
297
jklaus.append(klaus.__name__)
298
return self._preserve(klaus, jklaus)
300
return self.unpersistable("class %s deemed insecure" % str(klaus))
302
return self.unpersistable("class from module %s deemed insecure" % str(klaus.__module__))
305
def _jelly_dictionary(self, dict):
306
''' (internal) Jelly a dictionary.
307
returns a list of n items of the form (dictionary (attribute value) (attribute value) ...)
309
jdict = self._prepare(dict)
310
jdict.append(dictionary_atom)
311
for key, val in dict.items():
312
jkey = self.jelly(key)
313
jval = self.jelly(val)
314
jdict.append([jkey, jval])
315
return self._preserve(dict, jdict)
317
def _jelly_list(self, lst):
318
''' (internal) Jelly a list.
319
returns a list of n items of the form (list "value" "value" ...)
321
jlst = self._prepare(lst)
322
jlst.append(list_atom)
324
jlst.append(self.jelly(item))
325
return self._preserve(lst, jlst)
327
def _jelly_None(self, nne):
328
''' (internal) Jelly "None".
329
returns the list (none).
333
def _jelly_instance_method(self, im):
334
''' (internal) Jelly an instance method.
335
return a list of the form (method "name" (instance ...) (class ...))
337
jim = self._prepare(im)
339
jim.append(im.im_func.__name__)
340
jim.append(self.jelly(im.im_self))
341
jim.append(self.jelly(im.im_class))
342
return self._preserve(im, jim)
344
def _jelly_tuple(self, tup):
345
''' (internal) Jelly a tuple.
346
returns a list of n items of the form (tuple "value" "value" ...)
348
jtup = self._prepare(tup)
349
jtup.append(tuple_atom)
351
jtup.append(self.jelly(item))
352
return self._preserve(tup, jtup)
354
def _jelly_builtin_function_or_method(self, lst):
356
Jelly a builtin function. This is currently unimplemented.
358
raise 'currently unimplemented'
360
def _jelly_function(self, func):
361
''' (internal) Jelly a function.
362
Returns a list of the form (function "name" (module "name"))
365
module = sys.modules[pickle.whichmodule(func, name)]
366
if self.taster.isModuleAllowed(module.__name__):
367
jfunc = self._prepare(func)
368
jfunc.append(function_atom)
370
jfunc.append(self.jelly(module))
371
return self._preserve(func, jfunc)
373
return self.unpersistable("module %s deemed insecure" % str(module.__name__))
375
def _jelly_module(self, module):
377
Jelly a module. Return a list of the form (module "name")
379
if self.taster.isModuleAllowed(module.__name__):
380
jmod = self._prepare(module)
381
jmod.append(module_atom)
382
jmod.append(module.__name__)
383
return self._preserve(module, jmod)
385
return self.unpersistable("module %s deemed insecure" % str(module.__name__))
387
def unpersistable(self, reason, sxp=None):
389
Returns an sexp: (unpersistable "reason"). Utility method for making
390
note that a particular object could not be serialized.
394
sxp.append(unpersistable_atom)
402
def addDependant(self, mutableObject, key):
403
self.dependants.append( (mutableObject, key) )
405
def resolveDependants(self, newObject):
406
for mut, key in self.dependants:
408
if isinstance(newObject, NotKnown):
409
newObject.addDependant(mut, key)
412
assert 0, "I am not to be used as a dictionary key."
415
class _Tuple(NotKnown):
416
def __init__(self, l):
417
NotKnown.__init__(self)
420
for idx in xrange(len(l)):
421
if isinstance(l[idx], NotKnown):
422
self.locs.append(idx)
423
l[idx].addDependant(self, idx)
425
def __setitem__(self, n, obj):
427
if not isinstance(obj, NotKnown):
430
self.resolveDependants(tuple(self.l))
432
class _DictKeyAndValue:
433
def __init__(self, dict):
435
def __setitem__(self, n, obj):
437
raise AssertionError("DictKeyAndValue should only ever be called with 0 or 1")
442
if hasattr(self, "key") and hasattr(self, "value"):
443
self.dict[self.key] = self.value
446
class _Dereference(NotKnown):
447
def __init__(self, id):
448
NotKnown.__init__(self)
452
def __init__(self, taster, persistentLoad):
454
self.persistentLoad = persistentLoad
456
self.postCallbacks = []
458
def unjelly(self, obj):
459
o = self._unjelly(obj)
460
for m in self.postCallbacks:
464
def _unjelly(self, obj):
465
if type(obj) is not types.ListType:
468
if not self.taster.isTypeAllowed(jelType):
469
raise InsecureJelly(jelType)
470
thunk = getattr(self, '_unjelly_%s'%jelType)
474
def _unjelly_None(self, exp):
477
def _unjelly_unicode(self, exp):
478
return unicode(exp[0], "UTF-8")
480
def unjellyInto(self, obj, loc, jel):
481
o = self._unjelly(jel)
482
if isinstance(o, NotKnown):
483
o.addDependant(obj, loc)
487
def _unjelly_dereference(self, lst):
489
x = self.references.get(refid)
492
der = _Dereference(refid)
493
self.references[refid] = der
496
def _unjelly_reference(self, lst):
499
o = self._unjelly(exp)
500
ref = self.references.get(refid)
502
self.references[refid] = o
503
elif isinstance(ref, NotKnown):
504
ref.resolveDependants(o)
505
self.references[refid] = o
507
assert 0, "Multiple references with same ID!"
510
def _unjelly_tuple(self, lst):
514
if isinstance(self.unjellyInto(l, elem, lst[elem]), NotKnown):
521
def _unjelly_list(self, lst):
524
self.unjellyInto(l, elem, lst[elem])
527
def _unjelly_dictionary(self, lst):
530
kvd = _DictKeyAndValue(d)
531
self.unjellyInto(kvd, 0, k)
532
self.unjellyInto(kvd, 1, v)
536
def _unjelly_module(self, rest):
538
if type(moduleName) != types.StringType:
539
raise InsecureJelly("Attempted to unjelly a module with a non-string name.")
540
if not self.taster.isModuleAllowed(moduleName):
541
raise InsecureJelly("Attempted to unjelly module named %s" % repr(moduleName))
542
mod = __import__(moduleName, {}, {},"x")
545
def _unjelly_class(self, rest):
546
mod = self._unjelly(rest[0])
547
if type(mod) is not types.ModuleType:
548
raise InsecureJelly("class has a non-module module")
550
klaus = getattr(mod, name)
551
if type(klaus) is not types.ClassType:
552
raise InsecureJelly("class %s unjellied to something that isn't a class: %s" % (repr(name), repr(klaus)))
553
if not self.taster.isClassAllowed(klaus):
554
raise InsecureJelly("class not allowed: %s" % str(klaus))
557
def _unjelly_function(self, rest):
558
module = self._unjelly(rest[1])
559
if type(module) is not types.ModuleType:
560
raise InsecureJelly("function references a non-module module")
561
function = getattr(module, rest[0])
564
def _unjelly_persistent(self, rest):
565
if self.persistentLoad:
566
pload = self.persistentLoad(rest[0], self)
569
return Unpersistable("persistent callback not found")
571
def _unjelly_instance(self, rest):
572
clz = self._unjelly(rest[0])
573
if type(clz) is not types.ClassType:
574
raise InsecureJelly("Instance found with non-class class.")
575
if hasattr(clz, "__setstate__"):
576
inst = instance(clz, {})
577
state = self._unjelly(rest[1])
578
inst.__setstate__(state)
580
state = self._unjelly(rest[1])
581
inst = instance(clz, state)
582
if hasattr(clz, 'postUnjelly'):
583
self.postCallbacks.append(inst.postUnjelly)
586
def _unjelly_unpersistable(self, rest):
587
return Unpersistable(rest[0])
589
def _unjelly_method(self, rest):
590
''' (internal) unjelly a method
593
im_self = self._unjelly(rest[1])
594
im_class = self._unjelly(rest[2])
595
if im_class.__dict__.has_key(im_name):
597
im = getattr(im_class, im_name)
599
im = instancemethod(im_class.__dict__[im_name],
603
raise 'instance method changed'
609
Dummy class, used for unserializing instances.
615
#### Published Interface.
618
class InsecureJelly(Exception):
620
This exception will be raised when a jelly is deemed `insecure'; e.g. it
621
contains a type, class, or module disallowed by the specified `taster'
626
class DummySecurityOptions:
627
"""DummySecurityOptions() -> insecure security options
628
Dummy security options -- this class will allow anything.
630
def isModuleAllowed(self, moduleName):
631
"""DummySecurityOptions.isModuleAllowed(moduleName) -> boolean
632
returns 1 if a module by that name is allowed, 0 otherwise
636
def isClassAllowed(self, klass):
637
"""DummySecurityOptions.isClassAllowed(class) -> boolean
638
Assumes the module has already been allowed. Returns 1 if the given
639
class is allowed, 0 otherwise.
643
def isTypeAllowed(self, typeName):
644
"""DummySecurityOptions.isTypeAllowed(typeName) -> boolean
645
Returns 1 if the given type is allowed, 0 otherwise.
651
class SecurityOptions:
653
This will by default disallow everything, except for 'none'.
656
basicTypes = ["dictionary", "list", "tuple",
657
"reference", "dereference", "unpersistable",
664
# I don't believe any of these types can ever pose a security hazard,
665
# except perhaps "reference"...
666
self.allowedTypes = {"None": 1,
670
if hasattr(types, 'UnicodeType'):
671
self.allowedTypes['unicode'] = 1
672
self.allowedModules = {}
673
self.allowedClasses = {}
675
def allowBasicTypes(self):
676
"""SecurityOptions.allowBasicTypes()
677
Allow all `basic' types. (Dictionary and list. Int, string, and float are implicitly allowed.)
679
apply(self.allowTypes, self.basicTypes)
681
def allowTypes(self, *types):
682
"""SecurityOptions.allowTypes(typeString): Allow a particular type, by its name.
685
self.allowedTypes[string.replace(typ, ' ', '_')]=1
687
def allowInstancesOf(self, *classes):
688
"""SecurityOptions.allowInstances(klass, klass, ...): allow instances of the specified classes
689
This will also allow the 'instance', 'class', and 'module' types, as well as basic types.
691
self.allowBasicTypes()
692
self.allowTypes("instance", "class", "module")
693
for klass in classes:
694
self.allowModules(klass.__module__)
695
self.allowedClasses[klass] = 1
697
def allowModules(self, *modules):
698
"""SecurityOptions.allowModules(module, module, ...): allow modules by name
699
This will also allow the 'module' type.
701
for module in modules:
702
if type(module) == types.ModuleType:
703
module = module.__name__
704
self.allowedModules[module] = 1
706
def isModuleAllowed(self, moduleName):
707
"""SecurityOptions.isModuleAllowed(moduleName) -> boolean
708
returns 1 if a module by that name is allowed, 0 otherwise
710
return self.allowedModules.has_key(moduleName)
712
def isClassAllowed(self, klass):
713
"""SecurityOptions.isClassAllowed(class) -> boolean
714
Assumes the module has already been allowed. Returns 1 if the given
715
class is allowed, 0 otherwise.
717
return self.allowedClasses.has_key(klass)
719
def isTypeAllowed(self, typeName):
720
"""SecurityOptions.isTypeAllowed(typeName) -> boolean
721
Returns 1 if the given type is allowed, 0 otherwise.
723
return self.allowedTypes.has_key(typeName)
729
def jelly(object, taster = DummySecurityOptions(), persistentStore = None):
730
"""Serialize to s-expression.
732
Returns a list which is the serialized representation of an object. An
733
optional 'taster' argument takes a SecurityOptions and will mark any
734
insecure objects as unpersistable rather than serializing them.
736
return _Jellier(taster, persistentStore).jelly(object)
739
def unjelly(sexp, taster = DummySecurityOptions(), persistentLoad = None):
740
"""Unserialize from s-expression.
742
Takes an list that was the result from a call to jelly() and unserializes
743
an arbitrary object from it. The optional 'taster' argument, an instance
744
of SecurityOptions, will cause an InsecureJelly exception to be raised if a
745
disallowed type, module, or class attempted to unserialize.
747
return _Unjellier(taster, persistentLoad).unjelly(sexp)
added
removed
twisted/spread/jelly.py
