1
# -*- test-case-name: twisted.test.test_newjelly -*-
3
# Copyright (c) 2001-2004 Twisted Matrix Laboratories.
4
# See LICENSE for details.
7
"""S-expression-based persistence of python objects.
11
Future Plans: Optimization. Lots of optimization. No semantic breakages
12
should be necessary, but if small tweaks are required to gain acceptable
13
large-scale performance then they will be made. Although Glyph is the
14
maintainer, Bruce Mitchener will be supervising most of the optimization work
17
I do something very much like L{Pickle<pickle>}; however, pickle's main goal
18
seems to be efficiency (both in space and time); jelly's main goals are
19
security, human readability, and portability to other environments.
22
This is how Jelly converts various objects to s-expressions:
24
Boolean: True --> ['boolean', 'true']
28
List: [1, 2] --> ['list', 1, 2]
30
String: \"hello\" --> \"hello\"
34
Dictionary: {'a' : 1, 'b' : 'c'} --> ['dictionary', ['b', 'c'], ['a', 1]]
36
Module: UserString --> ['module', 'UserString']
38
Class: UserString.UserString --> ['class', ['module', 'UserString'], 'UserString']
40
Function: string.join --> ['function', 'join', ['module', 'string']]
42
Instance: s is an instance of UserString.UserString, with a __dict__ {'data': 'hello'}:
43
[\"UserString.UserString\", ['dictionary', ['data', 'hello']]]
45
# ['instance', ['class', ['module', 'UserString'], 'UserString'], ['dictionary', ['data', 'hello']]]
47
Class Method: UserString.UserString.center:
48
['method', 'center', ['None'], ['class', ['module', 'UserString'], 'UserString']]
50
Instance Method: s.center, where s is an instance of UserString.UserString:
51
['method', 'center', ['instance', ['reference', 1, ['class', ['module', 'UserString'], 'UserString']], ['dictionary', ['data', 'd']]], ['dereference', 1]]
53
@author: U{Glyph Lefkowitz<mailto:glyph@twistedmatrix.com>}
56
__version__ = "$Revision: 1.9 $"[11:-2]
63
from types import StringType
65
from types import UnicodeType
68
from types import IntType
69
from types import TupleType
70
from types import ListType
71
from types import DictType
72
from types import LongType
73
from types import FloatType
74
from types import FunctionType
75
from types import MethodType
76
from types import ModuleType
77
from types import DictionaryType
78
from types import InstanceType
79
from types import NoneType
80
from types import ClassType
84
from types import BooleanType
88
from new import instance
89
from new import instancemethod
93
from twisted.python.reflect import namedObject, namedModule, qual
94
from twisted.persisted.crefutil import NotKnown, _Tuple, _InstanceMethod, _DictKeyAndValue, _Dereference
95
from twisted.python import runtime
97
if runtime.platform.getType() == "java":
98
from org.python.core import PyStringMap
99
DictTypes = (DictionaryType, PyStringMap)
101
DictTypes = (DictionaryType,)
104
None_atom = "None" # N
106
class_atom = "class" # c
107
module_atom = "module" # m
108
function_atom = "function" # f
111
dereference_atom = 'dereference' # D
112
persistent_atom = 'persistent' # p
113
reference_atom = 'reference' # r
115
# mutable collections
116
dictionary_atom = "dictionary" # d
117
list_atom = 'list' # l
119
# immutable collections
120
# (assignment to __dict__ and __class__ still might go away!)
121
tuple_atom = "tuple" # t
125
unpersistable_atom = "unpersistable"# u
126
unjellyableRegistry = {}
127
unjellyableFactoryRegistry = {}
129
def _maybeClass(classnamep):
135
isObject = isinstance(classnamep, type)
136
if isinstance(classnamep, ClassType) or isObject:
137
return qual(classnamep)
140
def setUnjellyableForClass(classname, unjellyable):
141
"""Set which local class will represent a remote type.
143
If you have written a Copyable class that you expect your client to be
144
receiving, write a local "copy" class to represent it, then call::
146
jellier.setUnjellyableForClass('module.package.Class', MyJellier).
148
Call this at the module level immediately after its class
149
definition. MyCopier should be a subclass of RemoteCopy.
151
The classname may be a special tag returned by
152
'Copyable.getTypeToCopyFor' rather than an actual classname.
154
This call is also for cached classes, since there will be no
155
overlap. The rules are the same.
158
global unjellyableRegistry
159
classname = _maybeClass(classname)
160
unjellyableRegistry[classname] = unjellyable
161
globalSecurity.allowTypes(classname)
163
def setUnjellyableFactoryForClass(classname, copyFactory):
165
Set the factory to construct a remote instance of a type::
167
jellier.setFactoryForClass('module.package.Class', MyFactory)
169
Call this at the module level immediately after its class definition.
170
C{copyFactory} should return an instance or subclass of
171
L{RemoteCopy<pb.RemoteCopy>}.
173
Similar to L{setUnjellyableForClass} except it uses a factory instead
174
of creating an instance.
177
global unjellyableFactoryRegistry
178
classname = _maybeClass(classname)
179
unjellyableFactoryRegistry[classname] = copyFactory
180
globalSecurity.allowTypes(classname)
183
def setUnjellyableForClassTree(module, baseClass, prefix=None):
185
Set all classes in a module derived from C{baseClass} as copiers for
186
a corresponding remote class.
188
When you have a heirarchy of Copyable (or Cacheable) classes on
189
one side, and a mirror structure of Copied (or RemoteCache)
190
classes on the other, use this to setCopierForClass all your
191
Copieds for the Copyables.
193
Each copyTag (the \"classname\" argument to getTypeToCopyFor, and
194
what the Copyable's getTypeToCopyFor returns) is formed from
195
adding a prefix to the Copied's class name. The prefix defaults
196
to module.__name__. If you wish the copy tag to consist of solely
197
the classname, pass the empty string \'\'.
199
@param module: a module object from which to pull the Copied classes.
200
(passing sys.modules[__name__] might be useful)
202
@param baseClass: the base class from which all your Copied classes derive.
204
@param prefix: the string prefixed to classnames to form the
208
prefix = module.__name__
211
prefix = "%s." % prefix
213
for i in dir(module):
214
i_ = getattr(module, i)
215
if type(i_) == types.ClassType:
216
if issubclass(i_, baseClass):
217
setUnjellyableForClass('%s%s' % (prefix, i), i_)
219
def getInstanceState(inst, jellier):
220
"""Utility method to default to 'normal' state rules in serialization.
222
if hasattr(inst, "__getstate__"):
223
state = inst.__getstate__()
225
state = inst.__dict__
226
sxp = jellier.prepare(inst)
227
sxp.extend([qual(inst.__class__), jellier.jelly(state)])
228
return jellier.preserve(inst, sxp)
230
def setInstanceState(inst, unjellier, jellyList):
231
"""Utility method to default to 'normal' state rules in unserialization.
233
state = unjellier.unjelly(jellyList[1])
234
if hasattr(inst, "__setstate__"):
235
inst.__setstate__(state)
237
inst.__dict__ = state
242
This is an instance of a class that comes back when something couldn't be
245
def __init__(self, reason):
247
Initialize an unpersistable object with a descriptive `reason' string.
252
return "Unpersistable(%s)" % repr(self.reason)
255
"""Inherit from me to Jelly yourself directly.
257
def getStateFor(self, jellier):
260
def jellyFor(self, jellier):
261
sxp = jellier.prepare(self)
263
qual(self.__class__),
264
jellier.jelly(self.getStateFor(jellier))])
265
return jellier.preserve(self, sxp)
268
"""Inherit from me to Unjelly yourself directly.
270
def setStateFor(self, unjellier, state):
271
self.__dict__ = state
273
def unjellyFor(self, unjellier, jellyList):
274
state = unjellier.unjelly(jellyList[1])
275
self.setStateFor(unjellier, state)
279
"""(Internal) This class manages state for a call to jelly()
281
def __init__(self, taster, persistentStore, invoker):
285
self.seen = {} # maps from object ID to reference number
287
#self.persistentStore = persistentStore # ignored
288
self.invoker = invoker
290
constantTypes = {types.StringType : 1, types.IntType : 1,
291
types.FloatType : 1, types.LongType : 1}
293
# XXX ancient horrible backwards-compatibility
295
def prepare(self, obj):
298
def preserve(self, obj, jlist):
301
def _checkMutable(self, obj, refId):
303
if self.seen.has_key(objId):
304
objCheck, derefKey = self.seen[objId]
305
return [dereference_atom, derefKey]
306
self.seen[objId] = obj, refId
308
def jelly(self, obj):
309
if isinstance(obj, Jellyable):
312
preRef = self._checkMutable(obj, refId)
315
return obj.jellyFor(self)
317
if self.taster.isTypeAllowed(
318
string.replace(objType.__name__, ' ', '_')):
320
if ((objType is StringType) or
321
(objType is IntType) or
322
(objType is LongType) or
323
(objType is FloatType)):
329
if objType is MethodType:
331
obj.im_func.__name__,
332
self.jelly(obj.im_self),
333
self.jelly(obj.im_class)]
335
elif UnicodeType and objType is UnicodeType:
336
return ['unicode', obj.encode('UTF-8')]
337
elif objType is NoneType:
339
elif objType is FunctionType:
341
return ['function', str(pickle.whichmodule(obj, obj.__name__))
344
elif objType is ModuleType:
345
return ['module', obj.__name__]
346
elif objType is BooleanType:
347
return ['boolean', obj and 'true' or 'false']
348
elif objType is ClassType or issubclass(objType, type):
349
return ['class', qual(obj)]
352
preRef = self._checkMutable(obj, refId)
356
if objType is ListType:
357
sxp.append(list_atom)
359
sxp.append(self.jelly(item))
360
elif objType is TupleType:
361
sxp.append(tuple_atom)
363
sxp.append(self.jelly(item))
364
elif objType in DictTypes:
365
sxp.append(dictionary_atom)
366
for key, val in obj.items():
368
sxp.append([self.jelly(key), self.jelly(val)])
369
elif objType is InstanceType:
370
className = qual(obj.__class__)
371
if self.taster.isClassAllowed(obj.__class__):
372
sxp.append(className)
373
if hasattr(obj, "__getstate__"):
374
state = obj.__getstate__()
377
sxp.append(self.jelly(state))
380
"instance of class %s deemed insecure" %
381
qual(obj.__class__), sxp)
383
raise NotImplementedError("Don't know the type: %s" % objType)
386
if objType is types.InstanceType:
387
raise InsecureJelly("Class not allowed for instance: %s %s" %
388
(obj.__class__, obj))
389
raise InsecureJelly("Type not allowed for object: %s %s" %
392
def unpersistable(self, reason, sxp=None):
394
Returns an sexp: (unpersistable "reason"). Utility method for making
395
note that a particular object could not be serialized.
399
sxp.append(unpersistable_atom)
404
class NullReference(Exception):
406
This object is a marker for when a reference is made to an object that
407
can't be referenced, i.e.::
409
(tuple (tuple (reference 1)))
413
_theNullRef = NullReference()
416
def __init__(self, taster, persistentLoad, invoker):
418
#self.persistentLoad = persistentLoad # ignored
420
self.postCallbacks = []
421
self.invoker = invoker
423
def unjellyFull(self, obj):
424
o = self.unjelly(obj)
425
for m in self.postCallbacks:
429
def unjelly(self, obj):
430
if type(obj) is not types.ListType:
432
self.references.append(_theNullRef)
434
if not self.taster.isTypeAllowed(jelType):
435
raise InsecureJelly(jelType)
436
regClass = unjellyableRegistry.get(jelType)
437
if regClass is not None:
438
if isinstance(regClass, ClassType):
439
inst = _Dummy() # XXX chomp, chomp
440
self.resolveReference(inst)
441
inst.__class__ = regClass
442
val = inst.unjellyFor(self,obj)
444
refid = self.getRefId()
445
self.resolveReference(NotKnown())
446
val = regClass(self, obj) # this is how it ought to be done
447
self.resolveReference(val, refid)
448
if hasattr(val, 'postUnjelly'):
449
self.postCallbacks.append(inst.postUnjelly)
451
regFactory = unjellyableFactoryRegistry.get(jelType)
452
if regFactory is not None:
453
refid = self.getRefId()
454
self.resolveReference(NotKnown())
455
state = self.unjelly(obj[1])
456
inst = regFactory(state)
457
if hasattr(inst, 'postUnjelly'):
458
self.postCallbacks.append(inst.postUnjelly)
459
return self.resolveReference(inst, refid)
460
thunk = getattr(self, '_unjelly_%s'%jelType, None)
461
if thunk is not None:
464
nameSplit = string.split(jelType, '.')
465
modName = string.join(nameSplit[:-1], '.')
466
if not self.taster.isModuleAllowed(modName):
467
raise InsecureJelly("Module %s not allowed." % modName)
468
clz = namedObject(jelType)
469
if not self.taster.isClassAllowed(clz):
470
raise InsecureJelly("Class %s not allowed." % jelType)
471
if hasattr(clz, "__setstate__"):
472
ret = instance(clz, {})
473
self.resolveReference(ret)
474
state = self.unjelly(obj[1])
475
ret.__setstate__(state)
477
ret = instance(clz, {})
478
self.resolveReference(ret)
479
state = self.unjelly(obj[1])
481
if hasattr(clz, 'postUnjelly'):
482
self.postCallbacks.append(ret.postUnjelly)
485
def resolveReference(self, obj, index=-1):
486
if isinstance(self.references[index], NotKnown):
487
assert not isinstance(obj, NotKnown)
488
self.references[index].resolveDependants(obj)
489
self.references[index] = obj
492
def _unjelly_None(self, exp):
493
return self.resolveReference(None)
495
def _unjelly_unicode(self, exp):
497
return self.resolveReference(unicode(exp[0], "UTF-8"))
499
return self.resolveReference(Unpersistable(exp[0]))
501
def _unjelly_boolean(self, exp):
503
assert exp[0] in ('true', 'false')
504
return self.resolveReference(exp[0] == 'true')
506
return self.resolveReference(Unpersistable(exp[0]))
508
def unjellyInto(self, obj, loc, jel):
509
o = self.unjelly(jel)
510
if isinstance(o, NotKnown):
511
o.addDependant(obj, loc)
515
def _unjelly_dereference(self, lst):
517
return self.references[refid]
520
return len(self.references) - 1
522
def _unjelly_tuple(self, lst):
523
l = [NotKnown()] * len(lst)
526
refid = self.getRefId()
527
self.resolveReference(preTuple)
528
for elem in xrange(len(l)):
529
self.unjellyInto(preTuple, elem, lst[elem])
530
# zero-length tuples are false!!
531
# return preTuple.resolvedObject or preTuple
532
if preTuple.resolvedObject is None:
535
return preTuple.resolvedObject
537
def _unjelly_list(self, lst):
539
self.resolveReference(l)
541
self.unjellyInto(l, elem, lst[elem])
544
def _unjelly_dictionary(self, lst):
546
self.resolveReference(d)
548
self.references.append(_theNullRef)
549
kvd = _DictKeyAndValue(d)
550
self.unjellyInto(kvd, 0, k)
551
self.unjellyInto(kvd, 1, v)
555
def _unjelly_module(self, rest):
557
# if len(rest) > 0: warn("reference numbers will be out of sync")
558
if type(moduleName) != types.StringType:
559
raise InsecureJelly("Attempted to unjelly a module with a non-string name.")
560
if not self.taster.isModuleAllowed(moduleName):
561
raise InsecureJelly("Attempted to unjelly module named %s" % repr(moduleName))
562
mod = __import__(moduleName, {}, {},"x")
563
self.resolveReference(mod)
566
def _unjelly_class(self, rest):
567
clist = string.split(rest[0], '.')
568
# if len(rest) > 0: warn("reference numbers will be out of sync")
569
modName = string.join(clist[:-1], '.')
570
if not self.taster.isModuleAllowed(modName):
571
raise InsecureJelly("module %s not allowed" % modName)
572
klaus = namedObject(rest[0])
573
if type(klaus) is not types.ClassType:
574
raise InsecureJelly("class %s unjellied to something that isn't a class: %s" % (repr(name), repr(klaus)))
575
if not self.taster.isClassAllowed(klaus):
576
raise InsecureJelly("class not allowed: %s" % qual(klaus))
577
self.resolveReference(klaus)
580
def _unjelly_function(self, rest):
581
modSplit = string.split(rest[0], '.')
582
# if len(rest) > 0: warn("reference numbers will be out of sync")
583
modName = string.join(modSplit[:-1], '.')
584
if not self.taster.isModuleAllowed(modName):
585
raise InsecureJelly("Module not allowed: %s"% modName)
586
# XXX do I need an isFunctionAllowed?
587
function = namedObject(rest[0])
588
self.resolveReference(function)
591
def _unjelly_unpersistable(self, rest):
592
return self.resolveReference(Unpersistable(rest[0]))
594
def _unjelly_method(self, rest):
595
''' (internal) unjelly a method
598
rid = self.getRefId()
599
self.resolveReference(nk)
601
im_self = self.unjelly(rest[1])
602
im_class = self.unjelly(rest[2])
603
assert not isinstance(im_self, NotKnown)
604
if type(im_class) is not types.ClassType:
605
raise InsecureJelly("Method found with non-class class.")
606
if im_class.__dict__.has_key(im_name):
608
im = getattr(im_class, im_name)
610
im = instancemethod(im_class.__dict__[im_name],
614
raise 'instance method changed'
615
return self.resolveReference(im, rid)
620
Dummy class, used for unserializing instances.
626
#### Published Interface.
629
class InsecureJelly(Exception):
631
This exception will be raised when a jelly is deemed `insecure'; e.g. it
632
contains a type, class, or module disallowed by the specified `taster'
637
class DummySecurityOptions:
638
"""DummySecurityOptions() -> insecure security options
639
Dummy security options -- this class will allow anything.
641
def isModuleAllowed(self, moduleName):
642
"""DummySecurityOptions.isModuleAllowed(moduleName) -> boolean
643
returns 1 if a module by that name is allowed, 0 otherwise
647
def isClassAllowed(self, klass):
648
"""DummySecurityOptions.isClassAllowed(class) -> boolean
649
Assumes the module has already been allowed. Returns 1 if the given
650
class is allowed, 0 otherwise.
654
def isTypeAllowed(self, typeName):
655
"""DummySecurityOptions.isTypeAllowed(typeName) -> boolean
656
Returns 1 if the given type is allowed, 0 otherwise.
662
class SecurityOptions:
664
This will by default disallow everything, except for 'none'.
667
basicTypes = ["dictionary", "list", "tuple",
668
"reference", "dereference", "unpersistable",
669
"persistent", "long_int", "long", "dict"]
675
# I don't believe any of these types can ever pose a security hazard,
676
# except perhaps "reference"...
677
self.allowedTypes = {"None": 1,
685
if hasattr(types, 'UnicodeType'):
686
self.allowedTypes['unicode'] = 1
687
self.allowedModules = {}
688
self.allowedClasses = {}
690
def allowBasicTypes(self):
691
"""SecurityOptions.allowBasicTypes()
692
Allow all `basic' types. (Dictionary and list. Int, string, and float are implicitly allowed.)
694
self.allowTypes(*self.basicTypes)
696
def allowTypes(self, *types):
697
"""SecurityOptions.allowTypes(typeString): Allow a particular type, by its name.
700
self.allowedTypes[string.replace(typ, ' ', '_')]=1
702
def allowInstancesOf(self, *classes):
703
"""SecurityOptions.allowInstances(klass, klass, ...): allow instances
704
of the specified classes
706
This will also allow the 'instance', 'class' (renamed 'classobj' in
707
Python 2.3), and 'module' types, as well as basic types.
709
self.allowBasicTypes()
710
self.allowTypes("instance", "class", "classobj", "module")
711
for klass in classes:
712
self.allowTypes(qual(klass))
713
self.allowModules(klass.__module__)
714
self.allowedClasses[klass] = 1
716
def allowModules(self, *modules):
717
"""SecurityOptions.allowModules(module, module, ...): allow modules by name
718
This will also allow the 'module' type.
720
for module in modules:
721
if type(module) == types.ModuleType:
722
module = module.__name__
723
self.allowedModules[module] = 1
725
def isModuleAllowed(self, moduleName):
726
"""SecurityOptions.isModuleAllowed(moduleName) -> boolean
727
returns 1 if a module by that name is allowed, 0 otherwise
729
return self.allowedModules.has_key(moduleName)
731
def isClassAllowed(self, klass):
732
"""SecurityOptions.isClassAllowed(class) -> boolean
733
Assumes the module has already been allowed. Returns 1 if the given
734
class is allowed, 0 otherwise.
736
return self.allowedClasses.has_key(klass)
738
def isTypeAllowed(self, typeName):
739
"""SecurityOptions.isTypeAllowed(typeName) -> boolean
740
Returns 1 if the given type is allowed, 0 otherwise.
742
return (self.allowedTypes.has_key(typeName) or
746
globalSecurity = SecurityOptions()
747
globalSecurity.allowBasicTypes()
751
def jelly(object, taster = DummySecurityOptions(), persistentStore=None, invoker=None):
752
"""Serialize to s-expression.
754
Returns a list which is the serialized representation of an object. An
755
optional 'taster' argument takes a SecurityOptions and will mark any
756
insecure objects as unpersistable rather than serializing them.
758
jr = _Jellier(taster, persistentStore, invoker)
759
jel = jr.jelly(object)
760
# jr.__dict__.clear()
764
def unjelly(sexp, taster = DummySecurityOptions(), persistentLoad=None, invoker=None):
765
"""Unserialize from s-expression.
767
Takes an list that was the result from a call to jelly() and unserializes
768
an arbitrary object from it. The optional 'taster' argument, an instance
769
of SecurityOptions, will cause an InsecureJelly exception to be raised if a
770
disallowed type, module, or class attempted to unserialize.
772
ujr = _Unjellier(taster, persistentLoad, invoker)
773
result = ujr.unjellyFull(sexp)
774
# debugCrap.append(ujr.references)
added
removed
twisted/spread/newjelly.py
