3
/* Write Python objects to files and read them back.
4
This is intended for writing and reading compiled Python code only;
5
a true persistent storage facility would be much harder, since
6
it would have to take circular links and sharing into account. */
9
#include "longintrepr.h"
13
/* High water mark to determine when the marshalled object is dangerously deep
14
* and risks coring the interpreter. When the object stack gets this deep,
15
* raise an exception instead of continuing.
17
#define MAX_MARSHAL_STACK_DEPTH 5000
21
#define TYPE_STOPITER 'S'
22
#define TYPE_ELLIPSIS '.'
24
#define TYPE_INT64 'I'
25
#define TYPE_FLOAT 'f'
26
#define TYPE_COMPLEX 'x'
28
#define TYPE_STRING 's'
29
#define TYPE_TUPLE '('
33
#define TYPE_UNICODE 'u'
34
#define TYPE_UNKNOWN '?'
40
/* If fp == NULL, the following are valid: */
46
#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
47
else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
51
w_more(int c, WFILE *p)
55
return; /* An error already occurred */
56
size = PyString_Size(p->str);
57
newsize = size + 1024;
58
if (_PyString_Resize(&p->str, newsize) != 0) {
59
p->ptr = p->end = NULL;
62
p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
64
PyString_AS_STRING((PyStringObject *)p->str) + newsize;
65
*p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
70
w_string(char *s, int n, WFILE *p)
73
fwrite(s, 1, n, p->fp);
84
w_short(int x, WFILE *p)
87
w_byte((x>> 8) & 0xff, p);
91
w_long(long x, WFILE *p)
93
w_byte((int)( x & 0xff), p);
94
w_byte((int)((x>> 8) & 0xff), p);
95
w_byte((int)((x>>16) & 0xff), p);
96
w_byte((int)((x>>24) & 0xff), p);
101
w_long64(long x, WFILE *p)
109
w_object(PyObject *v, WFILE *p)
115
if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
118
else if (v == NULL) {
119
w_byte(TYPE_NULL, p);
121
else if (v == Py_None) {
122
w_byte(TYPE_NONE, p);
124
else if (v == PyExc_StopIteration) {
125
w_byte(TYPE_STOPITER, p);
127
else if (v == Py_Ellipsis) {
128
w_byte(TYPE_ELLIPSIS, p);
130
else if (PyInt_Check(v)) {
131
long x = PyInt_AS_LONG((PyIntObject *)v);
133
long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
135
w_byte(TYPE_INT64, p);
145
else if (PyLong_Check(v)) {
146
PyLongObject *ob = (PyLongObject *)v;
147
w_byte(TYPE_LONG, p);
152
for (i = 0; i < n; i++)
153
w_short(ob->ob_digit[i], p);
155
else if (PyFloat_Check(v)) {
156
char buf[256]; /* Plenty to format any double */
157
PyFloat_AsReprString(buf, (PyFloatObject *)v);
159
w_byte(TYPE_FLOAT, p);
163
#ifndef WITHOUT_COMPLEX
164
else if (PyComplex_Check(v)) {
165
char buf[256]; /* Plenty to format any double */
167
w_byte(TYPE_COMPLEX, p);
168
temp = (PyFloatObject*)PyFloat_FromDouble(
169
PyComplex_RealAsDouble(v));
170
PyFloat_AsReprString(buf, temp);
175
temp = (PyFloatObject*)PyFloat_FromDouble(
176
PyComplex_ImagAsDouble(v));
177
PyFloat_AsReprString(buf, temp);
184
else if (PyString_Check(v)) {
185
w_byte(TYPE_STRING, p);
186
n = PyString_GET_SIZE(v);
188
w_string(PyString_AS_STRING(v), n, p);
190
#ifdef Py_USING_UNICODE
191
else if (PyUnicode_Check(v)) {
193
utf8 = PyUnicode_AsUTF8String(v);
199
w_byte(TYPE_UNICODE, p);
200
n = PyString_GET_SIZE(utf8);
202
w_string(PyString_AS_STRING(utf8), n, p);
206
else if (PyTuple_Check(v)) {
207
w_byte(TYPE_TUPLE, p);
210
for (i = 0; i < n; i++) {
211
w_object(PyTuple_GET_ITEM(v, i), p);
214
else if (PyList_Check(v)) {
215
w_byte(TYPE_LIST, p);
216
n = PyList_GET_SIZE(v);
218
for (i = 0; i < n; i++) {
219
w_object(PyList_GET_ITEM(v, i), p);
222
else if (PyDict_Check(v)) {
224
PyObject *key, *value;
225
w_byte(TYPE_DICT, p);
226
/* This one is NULL object terminated! */
228
while (PyDict_Next(v, &pos, &key, &value)) {
232
w_object((PyObject *)NULL, p);
234
else if (PyCode_Check(v)) {
235
PyCodeObject *co = (PyCodeObject *)v;
236
w_byte(TYPE_CODE, p);
237
w_short(co->co_argcount, p);
238
w_short(co->co_nlocals, p);
239
w_short(co->co_stacksize, p);
240
w_short(co->co_flags, p);
241
w_object(co->co_code, p);
242
w_object(co->co_consts, p);
243
w_object(co->co_names, p);
244
w_object(co->co_varnames, p);
245
w_object(co->co_freevars, p);
246
w_object(co->co_cellvars, p);
247
w_object(co->co_filename, p);
248
w_object(co->co_name, p);
249
w_short(co->co_firstlineno, p);
250
w_object(co->co_lnotab, p);
252
else if (PyObject_CheckReadBuffer(v)) {
253
/* Write unknown buffer-style objects as a string */
255
PyBufferProcs *pb = v->ob_type->tp_as_buffer;
256
w_byte(TYPE_STRING, p);
257
n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
262
w_byte(TYPE_UNKNOWN, p);
270
PyMarshal_WriteLongToFile(long x, FILE *fp)
280
PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp)
289
typedef WFILE RFILE; /* Same struct with different invariants */
291
#define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
293
#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
296
r_string(char *s, int n, RFILE *p)
299
return fread(s, 1, n, p->fp);
300
if (p->end - p->ptr < n)
302
memcpy(s, p->ptr, n);
313
/* Sign-extension, in case short greater than 16 bits */
322
register FILE *fp = p->fp;
325
x |= (long)getc(fp) << 8;
326
x |= (long)getc(fp) << 16;
327
x |= (long)getc(fp) << 24;
331
x |= (long)rs_byte(p) << 8;
332
x |= (long)rs_byte(p) << 16;
333
x |= (long)rs_byte(p) << 24;
336
/* Sign extension for 64-bit machines */
337
x |= -(x & 0x80000000L);
342
/* r_long64 deals with the TYPE_INT64 code. On a machine with
343
sizeof(long) > 4, it returns a Python int object, else a Python long
344
object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
345
so there's no inefficiency here in returning a PyLong on 32-bit boxes
346
for everything written via TYPE_INT64 (i.e., if an int is written via
347
TYPE_INT64, it *needs* more than 32 bits).
352
long lo4 = r_long(p);
353
long hi4 = r_long(p);
355
long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
356
return PyInt_FromLong(x);
358
unsigned char buf[8];
360
int is_little_endian = (int)*(char*)&one;
361
if (is_little_endian) {
362
memcpy(buf, &lo4, 4);
363
memcpy(buf+4, &hi4, 4);
366
memcpy(buf, &hi4, 4);
367
memcpy(buf+4, &lo4, 4);
369
return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
378
int type = r_byte(p);
383
PyErr_SetString(PyExc_EOFError,
384
"EOF read where object expected");
395
Py_INCREF(PyExc_StopIteration);
396
return PyExc_StopIteration;
399
Py_INCREF(Py_Ellipsis);
403
return PyInt_FromLong(r_long(p));
414
ob = _PyLong_New(size);
418
for (i = 0; i < size; i++)
419
ob->ob_digit[i] = r_short(p);
420
return (PyObject *)ob;
428
if (r_string(buf, (int)n, p) != n) {
429
PyErr_SetString(PyExc_EOFError,
430
"EOF read where object expected");
434
PyFPE_START_PROTECT("atof", return 0)
436
PyFPE_END_PROTECT(dx)
437
return PyFloat_FromDouble(dx);
440
#ifndef WITHOUT_COMPLEX
446
if (r_string(buf, (int)n, p) != n) {
447
PyErr_SetString(PyExc_EOFError,
448
"EOF read where object expected");
452
PyFPE_START_PROTECT("atof", return 0)
456
if (r_string(buf, (int)n, p) != n) {
457
PyErr_SetString(PyExc_EOFError,
458
"EOF read where object expected");
462
PyFPE_START_PROTECT("atof", return 0)
465
return PyComplex_FromCComplex(c);
472
PyErr_SetString(PyExc_ValueError, "bad marshal data");
475
v = PyString_FromStringAndSize((char *)NULL, n);
477
if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
480
PyErr_SetString(PyExc_EOFError,
481
"EOF read where object expected");
486
#ifdef Py_USING_UNICODE
493
PyErr_SetString(PyExc_ValueError, "bad marshal data");
496
buffer = PyMem_NEW(char, n);
498
return PyErr_NoMemory();
499
if (r_string(buffer, (int)n, p) != n) {
501
PyErr_SetString(PyExc_EOFError,
502
"EOF read where object expected");
505
v = PyUnicode_DecodeUTF8(buffer, n, NULL);
514
PyErr_SetString(PyExc_ValueError, "bad marshal data");
517
v = PyTuple_New((int)n);
520
for (i = 0; i < n; i++) {
527
PyTuple_SET_ITEM(v, (int)i, v2);
534
PyErr_SetString(PyExc_ValueError, "bad marshal data");
537
v = PyList_New((int)n);
540
for (i = 0; i < n; i++) {
547
PyList_SetItem(v, (int)i, v2);
559
break; /* XXX Assume TYPE_NULL, not an error */
562
PyDict_SetItem(v, key, val);
569
if (PyEval_GetRestricted()) {
570
PyErr_SetString(PyExc_RuntimeError,
571
"cannot unmarshal code objects in "
572
"restricted execution mode");
576
int argcount = r_short(p);
577
int nlocals = r_short(p);
578
int stacksize = r_short(p);
579
int flags = r_short(p);
580
PyObject *code = NULL;
581
PyObject *consts = NULL;
582
PyObject *names = NULL;
583
PyObject *varnames = NULL;
584
PyObject *freevars = NULL;
585
PyObject *cellvars = NULL;
586
PyObject *filename = NULL;
587
PyObject *name = NULL;
589
PyObject *lnotab = NULL;
592
if (code) consts = r_object(p);
593
if (consts) names = r_object(p);
594
if (names) varnames = r_object(p);
595
#if MARSHAL_VERSION >= 21
596
if (varnames) freevars = r_object(p);
597
if (freevars) cellvars = r_object(p);
599
if (varnames) freevars = PyTuple_New(0);
600
if (freevars) cellvars = PyTuple_New(0);
602
if (cellvars) filename = r_object(p);
603
if (filename) name = r_object(p);
605
firstlineno = r_short(p);
606
lnotab = r_object(p);
609
if (!PyErr_Occurred()) {
610
v = (PyObject *) PyCode_New(
611
argcount, nlocals, stacksize, flags,
612
code, consts, names, varnames,
613
freevars, cellvars, filename, name,
614
firstlineno, lnotab);
621
Py_XDECREF(varnames);
622
Py_XDECREF(freevars);
623
Py_XDECREF(cellvars);
624
Py_XDECREF(filename);
632
/* Bogus data got written, which isn't ideal.
633
This will let you keep working and recover. */
634
PyErr_SetString(PyExc_ValueError, "bad marshal data");
641
PyMarshal_ReadShortFromFile(FILE *fp)
649
PyMarshal_ReadLongFromFile(FILE *fp)
657
/* Return size of file in bytes; < 0 if unknown. */
659
getfilesize(FILE *fp)
662
if (fstat(fileno(fp), &st) != 0)
669
/* If we can get the size of the file up-front, and it's reasonably small,
670
* read it in one gulp and delegate to ...FromString() instead. Much quicker
671
* than reading a byte at a time from file; speeds .pyc imports.
672
* CAUTION: since this may read the entire remainder of the file, don't
673
* call it unless you know you're done with the file.
676
PyMarshal_ReadLastObjectFromFile(FILE *fp)
678
/* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
679
* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
681
#define SMALL_FILE_LIMIT (1L << 14)
682
#define REASONABLE_FILE_LIMIT (1L << 18)
686
if (PyErr_Occurred()) {
687
fprintf(stderr, "XXX rd_object called with exception set\n");
691
filesize = getfilesize(fp);
693
char buf[SMALL_FILE_LIMIT];
695
if (filesize <= SMALL_FILE_LIMIT)
697
else if (filesize <= REASONABLE_FILE_LIMIT)
698
pBuf = (char *)PyMem_MALLOC(filesize);
701
size_t n = fread(pBuf, 1, filesize, fp);
702
v = PyMarshal_ReadObjectFromString(pBuf, n);
710
/* We don't have fstat, or we do but the file is larger than
711
* REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
713
return PyMarshal_ReadObjectFromFile(fp);
715
#undef SMALL_FILE_LIMIT
716
#undef REASONABLE_FILE_LIMIT
720
PyMarshal_ReadObjectFromFile(FILE *fp)
723
if (PyErr_Occurred()) {
724
fprintf(stderr, "XXX rd_object called with exception set\n");
728
return r_object(&rf);
732
PyMarshal_ReadObjectFromString(char *str, int len)
735
if (PyErr_Occurred()) {
736
fprintf(stderr, "XXX rds_object called with exception set\n");
743
return r_object(&rf);
747
PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */
751
wf.str = PyString_FromStringAndSize((char *)NULL, 50);
754
wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
755
wf.end = wf.ptr + PyString_Size(wf.str);
760
_PyString_Resize(&wf.str,
762
PyString_AS_STRING((PyStringObject *)wf.str)));
765
PyErr_SetString(PyExc_ValueError,
766
(wf.error==1)?"unmarshallable object"
767
:"object too deeply nested to marshal");
773
/* And an interface for Python programs... */
776
marshal_dump(PyObject *self, PyObject *args)
781
if (!PyArg_ParseTuple(args, "OO:dump", &x, &f))
783
if (!PyFile_Check(f)) {
784
PyErr_SetString(PyExc_TypeError,
785
"marshal.dump() 2nd arg must be file");
788
wf.fp = PyFile_AsFile(f);
790
wf.ptr = wf.end = NULL;
795
PyErr_SetString(PyExc_ValueError,
796
(wf.error==1)?"unmarshallable object"
797
:"object too deeply nested to marshal");
805
marshal_load(PyObject *self, PyObject *args)
810
if (!PyArg_ParseTuple(args, "O:load", &f))
812
if (!PyFile_Check(f)) {
813
PyErr_SetString(PyExc_TypeError,
814
"marshal.load() arg must be file");
817
rf.fp = PyFile_AsFile(f);
819
rf.ptr = rf.end = NULL;
822
if (PyErr_Occurred()) {
830
marshal_dumps(PyObject *self, PyObject *args)
833
if (!PyArg_ParseTuple(args, "O:dumps", &x))
835
return PyMarshal_WriteObjectToString(x);
839
marshal_loads(PyObject *self, PyObject *args)
845
if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
853
if (PyErr_Occurred()) {
860
static PyMethodDef marshal_methods[] = {
861
#ifndef MARSHAL_VERSION
862
{"dump", marshal_dump, 1},
863
{"load", marshal_load, 1},
864
{"dumps", marshal_dumps, 1},
865
{"loads", marshal_loads, 1},
867
{"load", marshal_load, 1},
868
{"loads", marshal_loads, 1},
870
{NULL, NULL} /* sentinel */
873
#if MARSHAL_VERSION == 20
875
initmarshal_20 (void)
877
(void) Py_InitModule("marshal_20" , marshal_methods);
880
##elif MARSHAL_VERSION == 22
882
#initmarshal_22 (void)
884
# (void) Py_InitModule("marshal_22" , marshal_methods);
891
(void) Py_InitModule("marshal", marshal_methods);