« back to all changes in this revision
Viewing changes to meliae/_scanner.pyx
- browse files at revision 184
- compare with another revision
- download diff
- download tarball
- view history from revision 184
- Committer: John Arbash Meinel
- Date: 2010-08-10 16:21:52 UTC
- mfrom: (183.1.6 special_cases)
- Revision ID: john@arbash-meinel.com-20100810162152-kwinrlv78flsdox5
We now support custom sizeof methods registered at runtime.
added
removed
meliae/_scanner.pyx
39
39
void _dump_object_info(write_callback write, void *callee_data,
40
40
object c_obj, object nodump, int recurse)
41
41
object _get_referents(object c_obj)
42
object _get_special_case_dict()
42
43
43
44
44
45
_word_size = sizeof(Py_ssize_t)
111
112
tp_traverse.
112
113
"""
113
114
return _get_referents(obj)
115
116
117
def add_special_size(object tp_name, object size_of_32, object size_of_64):
118
"""Special case a given object size.
119
120
This is only meant to be used for objects we don't already handle or which
121
don't implement __sizeof__ (those are checked before this check happens).
122
123
This is meant for things like zlib.Compress which allocates a lot of
124
internal buffers, which are not easily accessible (but can be
125
approximated). The gc header should not be included in this size, it will
126
be added at runtime.
127
128
Setting the value to None will remove the value.
129
130
(We only distinguish size_of_32 from size_of_64 for the implementer's
131
benefit, since sizeof() is not generally accessible from Python.)
132
133
:param tp_name: The type string we care about (such as 'zlib.Compress').
134
This will be matched against object->type->tp_name.
135
:param size_of_32: Called when _word_size == 32-bits
136
:param size_of_64: Called when _word_size == 64-bits
137
:return: None
138
"""
139
special_dict = _get_special_case_dict()
140
if _word_size == 4:
141
sz = size_of_32
142
elif _word_size == 8:
143
sz = size_of_64
144
else:
145
raise RuntimeError('Unknown word size: %s' % (_word_size,))
146
if sz is None:
147
if tp_name in special_dict:
148
del special_dict[tp_name]
149
else:
150
special_dict[tp_name] = sz
151
152
153
def _zlib_size_of_32(zlib_obj):
154
"""Return a __sizeof__ for a zlib object."""
155
cdef Py_ssize_t size
156
157
t = type(zlib_obj)
158
name = t.__name__
159
# Size of the zlib 'compobject', (PyObject_HEAD + z_stream, + misc)
160
size = 56
161
if name.endswith('Decompress'):
162
# _get_referents doesn't track into these attributes, so we just
163
# attribute the size to the object itself.
164
size += _size_of(zlib_obj.unused_data)
165
size += _size_of(zlib_obj.unconsumed_tail)
166
# sizeof(inflate_state)
167
size += 7116
168
# sizeof(buffers allocated for inflate)
169
# (1 << state->wbits)
170
# However, we don't have access to wbits, so we assume the default (and
171
# largest) of 15 wbits
172
size += (1 << 15)
173
# Empirically 42kB / object during decompression, and this gives 39998
174
elif name.endswith('Compress'):
175
# compress objects have a reference to unused_data, etc, but it always
176
# points to the empty string.
177
# sizeof(deflate_state)
178
size += 5828
179
# We don't have access to the stream C attributes, so we assume the
180
# standard values and go with it
181
# Pos == unsigned short
182
# Byte == unsigned char
183
# w_size = 1 << s->w_bits, default 15 => (1<<15)
184
# memLevel default is 8 (maybe 9?)
185
# s->w_size * 2*sizeof(Byte) = (1<<15) * 2 * 1 = 65536
186
size += 65536
187
# s_>w_size * sizeof(Pos) = (1<<15) * 2 = 65536
188
size += 65536
189
# s->hash_size * sizeof(Pos) = (1 << (8+7)) * 2 = 65536
190
size += 65536
191
# s->lit_bufsize = 1 << (8 + 6) = (1 << 14) = 16384
192
# s->pending_buf = lit_bufsize * (sizeof(ush)+2) = 4*16384 = 65536
193
size += 65536
194
# empirically, I got ~96378 bytes/object after allocating a lot of them
195
# After sending a bunch of compression data to all of them, I got
196
# ~270127 bytes/object. (according to WorkingMem)
197
# This gives 268028, which is pretty close
198
else:
199
return -1
200
# We assume that everything is at least aligned to word boundary
201
if size % _word_size != 0:
202
size += _word_size - (size % _word_size)
203
return size
204
205
206
def _zlib_size_of_64(zlib_obj):
207
"""Return a __sizeof__ for a zlib object."""
208
t = type(zlib_obj)
209
name = t.__name__
210
# Size of the zlib 'compobject', (PyObject_HEAD + z_stream, + misc)
211
# All the 64-bit numbers here are 'made up'
212
size = (56 * 2)
213
if name.endswith('Decompress'):
214
size += _size_of(zlib_obj.unused_data)
215
size += _size_of(zlib_obj.unconsumed_tail)
216
# sizeof(inflate_state)
217
size += (7116 * 2)
218
# sizeof(buffers allocated for inflate)
219
# (1 << state->wbits)
220
# However, we don't have access to wbits, so we assume the default (and
221
# largest) of 15 wbits
222
size += (1 << 15)
223
elif name.endswith('Compress'):
224
# sizeof(deflate_state)
225
size += (5828 * 2)
226
# We don't have access to the stream C attributes, so we assume the
227
# standard values and go with it
228
# s->w_size * 2*sizeof(Byte) = (1<<15) * 2 * 1 = 65536
229
size += 65536
230
# s_>w_size * sizeof(Pos) = (1<<15) * 2 = 65536
231
size += 65536
232
# s->hash_size * sizeof(Pos) = (1 << (8+7)) * 2 = 65536
233
size += 65536
234
# s->lit_bufsize = 1 << (8 + 6) = (1 << 14) = 16384
235
# s->pending_buf = lit_bufsize * (sizeof(ush)+2) = 4*16384 = 65536
236
size += 65536
237
else:
238
return -1
239
if size % _word_size != 0:
240
size += _word_size - (size % _word_size)
241
return size
242
243
add_special_size('zlib.Compress', _zlib_size_of_32, _zlib_size_of_64)
244
add_special_size('zlib.Decompress', _zlib_size_of_32, _zlib_size_of_64)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1