« back to all changes in this revision
Viewing changes to meliae/_scanner_core.c
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: Bazaar Package Importer
- Author(s): Jelmer Vernooij
- Date: 2009-12-19 18:23:37 UTC
- Revision ID: james.westby@ubuntu.com-20091219182337-t09txw6ca1yfysn9
Tags: upstream-0.2.0
Import upstream version 0.2.0
- files added:
- meliae
- meliae/tests
added
removed
meliae/_scanner_core.c
1
/* Copyright (C) 2009 Canonical Ltd
2
*
3
* This program is free software: you can redistribute it and/or modify
4
* it under the terms of the GNU General Public License version 3 as
5
* published by the Free Software Foundation.
6
*
7
* This program is distributed in the hope that it will be useful, but
8
* WITHOUT ANY WARRANTY; without even the implied warranty of
9
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
* General Public License for more details.
11
*
12
* You should have received a copy of the GNU General Public License
13
* along with this program. If not, see <http://www.gnu.org/licenses/>.
14
*/
15
16
/* The core of parsing is split into a pure C module, so that we can guarantee
17
* that we won't be creating objects in the internal loops.
18
*/
19
20
#include "_scanner_core.h"
21
22
#ifndef Py_TYPE
23
# define Py_TYPE(o) ((o)->ob_type)
24
#endif
25
26
// %zd is the gcc convention for defining that we are formatting a size_t
27
// object, windows seems to prefer %ld, though perhaps we need to first check
28
// sizeof(size_t) ?
29
#ifdef _WIN32
30
# if defined(_M_X64) || defined(__amd64__)
31
# define SSIZET_FMT "%ld"
32
# else
33
# define SSIZET_FMT "%d"
34
# endif
35
# define snprintf _snprintf
36
#else
37
# define SSIZET_FMT "%zd"
38
#endif
39
40
#if defined(__GNUC__)
41
# define inline __inline__
42
#elif defined(_MSC_VER)
43
# define inline __inline
44
#else
45
# define inline
46
#endif
47
48
struct ref_info {
49
write_callback write;
50
void *data;
51
int first;
52
PyObject *nodump;
53
};
54
55
void _dump_object_to_ref_info(struct ref_info *info, PyObject *c_obj,
56
int recurse);
57
#ifdef __GNUC__
58
static void _write_to_ref_info(struct ref_info *info, const char *fmt_string, ...)
59
__attribute__((format(printf, 2, 3)));
60
#else
61
static void _write_to_ref_info(struct ref_info *info, const char *fmt_string, ...);
62
#endif
63
64
/* The address of the last thing we dumped. Stuff like dumping the string
65
* interned dictionary will dump the same string 2x in a row. This helps
66
* prevent that.
67
*/
68
static PyObject *_last_dumped = NULL;
69
70
void
71
_clear_last_dumped()
72
{
73
_last_dumped = NULL;
74
}
75
76
Py_ssize_t
77
_basic_object_size(PyObject *c_obj)
78
{
79
Py_ssize_t size;
80
size = c_obj->ob_type->tp_basicsize;
81
if (PyType_HasFeature(c_obj->ob_type, Py_TPFLAGS_HAVE_GC)) {
82
size += sizeof(PyGC_Head);
83
}
84
return size;
85
}
86
87
88
Py_ssize_t
89
_var_object_size(PyVarObject *c_obj)
90
{
91
Py_ssize_t num_entries;
92
num_entries = PyObject_Size((PyObject *)c_obj);
93
if (num_entries < 0) {
94
/* This object doesn't support len() */
95
num_entries = 0;
96
PyErr_Clear();
97
}
98
return _basic_object_size((PyObject *)c_obj)
99
+ num_entries * c_obj->ob_type->tp_itemsize;
100
}
101
102
Py_ssize_t
103
_size_of_from__sizeof__(PyObject *c_obj)
104
{
105
PyObject *size_obj = NULL;
106
Py_ssize_t size = -1;
107
108
if (PyType_CheckExact(c_obj)) {
109
// Types themselves may have a __sizeof__ attribute, but it is the
110
// unbound method, which takes an instance
111
return -1;
112
}
113
size_obj = PyObject_CallMethod(c_obj, "__sizeof__", NULL);
114
if (size_obj == NULL) {
115
// Not sure what happened, but this won't work, it could be a simple
116
// attribute error, or it could be something else.
117
PyErr_Clear();
118
return -1;
119
}
120
size = PyInt_AsSsize_t(size_obj);
121
if (size == -1) {
122
// Probably an error occurred, we don't know for sure, but we might as
123
// well just claim that we don't know the size. We *could* check
124
// PyErr_Occurred(), but if we are just clearing it anyway...
125
PyErr_Clear();
126
return -1;
127
}
128
// There is one trick left. Namely, __sizeof__ doesn't seem to include the
129
// GC overhead, so let's add that back in
130
if (PyType_HasFeature(c_obj->ob_type, Py_TPFLAGS_HAVE_GC)) {
131
size += sizeof(PyGC_Head);
132
}
133
return size;
134
}
135
136
137
Py_ssize_t
138
_size_of_list(PyListObject *c_obj)
139
{
140
Py_ssize_t size;
141
size = _basic_object_size((PyObject *)c_obj);
142
size += sizeof(PyObject*) * c_obj->allocated;
143
return size;
144
}
145
146
147
Py_ssize_t
148
_size_of_set(PySetObject *c_obj)
149
{
150
Py_ssize_t size;
151
size = _basic_object_size((PyObject *)c_obj);
152
if (c_obj->table != c_obj->smalltable) {
153
size += sizeof(setentry) * (c_obj->mask + 1);
154
}
155
return size;
156
}
157
158
159
Py_ssize_t
160
_size_of_dict(PyDictObject *c_obj)
161
{
162
Py_ssize_t size;
163
size = _basic_object_size((PyObject *)c_obj);
164
if (c_obj->ma_table != c_obj->ma_smalltable) {
165
size += sizeof(PyDictEntry) * (c_obj->ma_mask + 1);
166
}
167
return size;
168
}
169
170
171
Py_ssize_t
172
_size_of_unicode(PyUnicodeObject *c_obj)
173
{
174
Py_ssize_t size;
175
size = _basic_object_size((PyObject *)c_obj);
176
size += Py_UNICODE_SIZE * c_obj->length;
177
return size;
178
}
179
180
181
Py_ssize_t
182
_size_of(PyObject *c_obj)
183
{
184
Py_ssize_t size;
185
186
if PyList_Check(c_obj) {
187
return _size_of_list((PyListObject *)c_obj);
188
} else if PyAnySet_Check(c_obj) {
189
return _size_of_set((PySetObject *)c_obj);
190
} else if PyDict_Check(c_obj) {
191
return _size_of_dict((PyDictObject *)c_obj);
192
} else if PyUnicode_Check(c_obj) {
193
return _size_of_unicode((PyUnicodeObject *)c_obj);
194
}
195
196
size = _size_of_from__sizeof__(c_obj);
197
if (size != -1) {
198
return size;
199
}
200
201
if (c_obj->ob_type->tp_itemsize != 0) {
202
// Variable length object with inline storage
203
// total size is tp_itemsize * ob_size
204
return _var_object_size((PyVarObject *)c_obj);
205
}
206
return _basic_object_size(c_obj);
207
}
208
209
210
static void
211
_write_to_ref_info(struct ref_info *info, const char *fmt_string, ...)
212
{
213
char temp_buf[1024] = {0};
214
va_list args;
215
size_t n_bytes;
216
217
va_start(args, fmt_string);
218
n_bytes = vsnprintf(temp_buf, 1024, fmt_string, args);
219
va_end(args);
220
info->write(info->data, temp_buf, n_bytes);
221
}
222
223
224
static inline void
225
_write_static_to_info(struct ref_info *info, const char data[])
226
{
227
/* These are static strings, do we need to do strlen() each time? */
228
info->write(info->data, data, strlen(data));
229
}
230
231
int
232
_dump_reference(PyObject *c_obj, void* val)
233
{
234
struct ref_info *info;
235
size_t n_bytes;
236
char buf[24] = {0}; /* it seems that 64-bit long fits in 20 decimals */
237
238
info = (struct ref_info*)val;
239
/* TODO: This is casting a pointer into an unsigned long, which we assume
240
* is 'long enough'. We probably should really be using uintptr_t or
241
* something like that.
242
*/
243
if (info->first) {
244
info->first = 0;
245
n_bytes = snprintf(buf, 24, "%lu", (unsigned long)c_obj);
246
} else {
247
n_bytes = snprintf(buf, 24, ", %lu", (unsigned long)c_obj);
248
}
249
info->write(info->data, buf, n_bytes);
250
return 0;
251
}
252
253
254
int
255
_dump_child(PyObject *c_obj, void *val)
256
{
257
struct ref_info *info;
258
info = (struct ref_info *)val;
259
// The caller has asked us to dump self, but no recursive children
260
_dump_object_to_ref_info(info, c_obj, 0);
261
return 0;
262
}
263
264
265
int
266
_dump_if_no_traverse(PyObject *c_obj, void *val)
267
{
268
struct ref_info *info;
269
info = (struct ref_info *)val;
270
/* Objects without traverse are simple things without refs, and built-in
271
* types have a traverse, but they won't be part of gc.get_objects().
272
*/
273
if (Py_TYPE(c_obj)->tp_traverse == NULL
274
|| (PyType_Check(c_obj)
275
&& !PyType_HasFeature((PyTypeObject*)c_obj, Py_TPFLAGS_HEAPTYPE)))
276
{
277
_dump_object_to_ref_info(info, c_obj, 0);
278
} else if (!PyType_HasFeature(Py_TYPE(c_obj), Py_TPFLAGS_HAVE_GC)) {
279
/* This object is not considered part of the garbage collector, even
280
* if it does [not] have a tp_traverse function.
281
*/
282
_dump_object_to_ref_info(info, c_obj, 1);
283
}
284
return 0;
285
}
286
287
288
static inline void
289
_dump_json_c_string(struct ref_info *info, const char *buf, Py_ssize_t len)
290
{
291
Py_ssize_t i;
292
char c, *ptr, *end;
293
char out_buf[1024] = {0};
294
295
// Never try to dump more than 100 chars
296
if (len == -1) {
297
len = strlen(buf);
298
}
299
if (len > 100) {
300
len = 100;
301
}
302
ptr = out_buf;
303
end = out_buf + 1024;
304
*ptr++ = '"';
305
for (i = 0; i < len; ++i) {
306
c = buf[i];
307
if (c <= 0x1f || c > 0x7e) { // use the unicode escape sequence
308
ptr += snprintf(ptr, end-ptr, "\\u00%02x",
309
((unsigned short)c & 0xFF));
310
} else if (c == '\\' || c == '/' || c == '"') {
311
*ptr++ = '\\';
312
*ptr++ = c;
313
} else {
314
*ptr++ = c;
315
}
316
}
317
*ptr++ = '"';
318
if (ptr >= end) {
319
/* Abort somehow */
320
}
321
info->write(info->data, out_buf, ptr-out_buf);
322
}
323
324
void
325
_dump_string(struct ref_info *info, PyObject *c_obj)
326
{
327
Py_ssize_t str_size;
328
char *str_buf;
329
330
str_buf = PyString_AS_STRING(c_obj);
331
str_size = PyString_GET_SIZE(c_obj);
332
333
_dump_json_c_string(info, str_buf, str_size);
334
}
335
336
337
void
338
_dump_unicode(struct ref_info *info, PyObject *c_obj)
339
{
340
// TODO: consider writing to a small memory buffer, before writing to disk
341
Py_ssize_t uni_size;
342
Py_UNICODE *uni_buf, c;
343
Py_ssize_t i;
344
char out_buf[1024] = {0}, *ptr, *end;
345
346
uni_buf = PyUnicode_AS_UNICODE(c_obj);
347
uni_size = PyUnicode_GET_SIZE(c_obj);
348
349
// Never try to dump more than this many chars
350
if (uni_size > 100) {
351
uni_size = 100;
352
}
353
ptr = out_buf;
354
end = out_buf + 1024;
355
*ptr++ = '"';
356
for (i = 0; i < uni_size; ++i) {
357
c = uni_buf[i];
358
if (c <= 0x1f || c > 0x7e) {
359
ptr += snprintf(ptr, end-ptr, "\\u%04x",
360
((unsigned short)c & 0xFFFF));
361
} else if (c == '\\' || c == '/' || c == '"') {
362
*ptr++ = '\\';
363
*ptr++ = (char)c;
364
} else {
365
*ptr++ = (char)c;
366
}
367
}
368
*ptr++ = '"';
369
if (ptr >= end) {
370
/* We should fail here */
371
}
372
info->write(info->data, out_buf, ptr-out_buf);
373
}
374
375
376
void
377
_dump_object_info(write_callback write, void *callee_data,
378
PyObject *c_obj, PyObject *nodump, int recurse)
379
{
380
struct ref_info info;
381
382
info.write = write;
383
info.data = callee_data;
384
info.first = 1;
385
info.nodump = nodump;
386
if (nodump != NULL) {
387
Py_INCREF(nodump);
388
}
389
_dump_object_to_ref_info(&info, c_obj, recurse);
390
if (info.nodump != NULL) {
391
Py_DECREF(nodump);
392
}
393
}
394
395
void
396
_dump_object_to_ref_info(struct ref_info *info, PyObject *c_obj, int recurse)
397
{
398
Py_ssize_t size;
399
int retval;
400
401
if (info->nodump != NULL &&
402
info->nodump != Py_None
403
&& PyAnySet_Check(info->nodump))
404
{
405
if (c_obj == info->nodump) {
406
/* Don't dump the 'nodump' set. */
407
return;
408
}
409
/* note this isn't exactly what we want. It checks for equality, not
410
* the exact object. However, for what it is used for, it is often
411
* 'close enough'.
412
*/
413
retval = PySet_Contains(info->nodump, c_obj);
414
if (retval == 1) {
415
/* This object is part of the no-dump set, don't dump the object */
416
return;
417
} else if (retval == -1) {
418
/* An error was raised, but we don't care, ignore it */
419
PyErr_Clear();
420
}
421
}
422
423
if (c_obj == _last_dumped) {
424
/* We just dumped this object, no need to do it again. */
425
return;
426
}
427
_last_dumped = c_obj;
428
size = _size_of(c_obj);
429
_write_to_ref_info(info, "{\"address\": %lu, \"type\": ",
430
(unsigned long)c_obj);
431
_dump_json_c_string(info, c_obj->ob_type->tp_name, -1);
432
_write_to_ref_info(info, ", \"size\": " SSIZET_FMT, _size_of(c_obj));
433
// HANDLE __name__
434
if (PyModule_Check(c_obj)) {
435
_write_static_to_info(info, ", \"name\": ");
436
_dump_json_c_string(info, PyModule_GetName(c_obj), -1);
437
} else if (PyFunction_Check(c_obj)) {
438
_write_static_to_info(info, ", \"name\": ");
439
_dump_string(info, ((PyFunctionObject *)c_obj)->func_name);
440
} else if (PyType_Check(c_obj)) {
441
_write_static_to_info(info, ", \"name\": ");
442
_dump_json_c_string(info, ((PyTypeObject *)c_obj)->tp_name, -1);
443
} else if (PyClass_Check(c_obj)) {
444
/* Old style class */
445
_write_static_to_info(info, ", \"name\": ");
446
_dump_string(info, ((PyClassObject *)c_obj)->cl_name);
447
}
448
if (PyString_Check(c_obj)) {
449
_write_to_ref_info(info, ", \"len\": " SSIZET_FMT, PyString_GET_SIZE(c_obj));
450
_write_static_to_info(info, ", \"value\": ");
451
_dump_string(info, c_obj);
452
} else if (PyUnicode_Check(c_obj)) {
453
_write_to_ref_info(info, ", \"len\": " SSIZET_FMT, PyUnicode_GET_SIZE(c_obj));
454
_write_static_to_info(info, ", \"value\": ");
455
_dump_unicode(info, c_obj);
456
} else if (PyBool_Check(c_obj)) {
457
if (c_obj == Py_True) {
458
_write_static_to_info(info, ", \"value\": \"True\"");
459
} else if (c_obj == Py_False) {
460
_write_static_to_info(info, ", \"value\": \"False\"");
461
} else {
462
_write_to_ref_info(info, ", \"value\": %ld", PyInt_AS_LONG(c_obj));
463
}
464
} else if (PyInt_CheckExact(c_obj)) {
465
_write_to_ref_info(info, ", \"value\": %ld", PyInt_AS_LONG(c_obj));
466
} else if (PyTuple_Check(c_obj)) {
467
_write_to_ref_info(info, ", \"len\": " SSIZET_FMT, PyTuple_GET_SIZE(c_obj));
468
} else if (PyList_Check(c_obj)) {
469
_write_to_ref_info(info, ", \"len\": " SSIZET_FMT, PyList_GET_SIZE(c_obj));
470
} else if (PyAnySet_Check(c_obj)) {
471
_write_to_ref_info(info, ", \"len\": " SSIZET_FMT, PySet_GET_SIZE(c_obj));
472
} else if (PyDict_Check(c_obj)) {
473
_write_to_ref_info(info, ", \"len\": " SSIZET_FMT, PyDict_Size(c_obj));
474
}
475
_write_static_to_info(info, ", \"refs\": [");
476
if (Py_TYPE(c_obj)->tp_traverse != NULL) {
477
info->first = 1;
478
Py_TYPE(c_obj)->tp_traverse(c_obj, _dump_reference, info);
479
}
480
_write_static_to_info(info, "]}\n");
481
if (Py_TYPE(c_obj)->tp_traverse != NULL && recurse != 0) {
482
if (recurse == 2) { /* Always dump one layer deeper */
483
Py_TYPE(c_obj)->tp_traverse(c_obj, _dump_child, info);
484
} else if (recurse == 1) {
485
/* strings and such aren't in gc.get_objects, so we need to dump
486
* them when they are referenced.
487
*/
488
Py_TYPE(c_obj)->tp_traverse(c_obj, _dump_if_no_traverse, info);
489
}
490
}
491
}
492
493
static int
494
_append_object(PyObject *visiting, void* data)
495
{
496
PyObject *lst;
497
lst = (PyObject *)data;
498
if (lst == NULL) {
499
return -1;
500
}
501
if (PyList_Append(data, visiting) == -1) {
502
return -1;
503
}
504
return 0;
505
}
506
/**
507
* Return a PyList of all objects referenced via tp_traverse.
508
*/
509
PyObject *_get_referents(PyObject *c_obj)
510
{
511
PyObject *lst;
512
513
lst = PyList_New(0);
514
if (lst == NULL) {
515
return NULL;
516
}
517
if (Py_TYPE(c_obj)->tp_traverse != NULL) {
518
Py_TYPE(c_obj)->tp_traverse(c_obj, _append_object, lst);
519
}
520
return lst;
521
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.1