1
/* Copyright (C) 2001-2006 Artifex Software, Inc.
4
This software is provided AS-IS with no warranty, either express or
7
This software is distributed under license and may not be copied, modified
8
or distributed except as expressly authorized under the terms of that
9
license. Refer to licensing information at http://www.artifex.com/
10
or contact Artifex Software, Inc., 7 Mt. Lassen Drive - Suite A-134,
11
San Rafael, CA 94903, U.S.A., +1(415)492-9861, for further information.
14
/* $Id: gxclmem.c 8721 2008-05-09 02:18:14Z ray $ */
15
/* RAM-based command list implementation */
22
* Based on: memfile.c Version: 1.4 3/21/95 14:59:33 by Ray Johnston.
23
* Copyright assigned to Aladdin Enterprises.
26
/*****************************************************************************
28
This package is more or less optimal for use by the clist routines, with
29
a couple of the more likely to change "tuning" parameters given in the
30
two macros below -- NEED_TO_COMPRESS and GET_NUM_RAW_BUFFERS. Usually
31
the NEED_TO_COMPRESS decision will be deferred as long as possible based
32
on some total system free RAM space remaining.
34
The data structures are in "memfile.h", and the primary 'tuning' parameter
35
is MEMFILE_DATA_SIZE. This should not be too small to keep the overhead
36
ratio of the block structures to the clist data small. A value of 16384
37
is probably in the ballpark.
39
The concept is that a memory based "file" is created initially without
40
compression, with index blocks every MEMFILE_DATA_SIZE of the file. The
41
primary blocks (used by the memfile_fseek logic) for indexing into the
42
file are called 'logical' (LOG_MEMFILE_BLK) and the data in stored in a
43
different block called a 'physical' block (PHYS_MEMFILE_BLK). When the
44
file is not yet compressed, indicated by (f->phys_curr==NULL), then there
45
is one physical block for each logical block. The physical block also has
46
the 'data_limit' set to NULL if the data is not compressed. Thus when a
47
file is not compressed there is one physical block for each logical block.
51
When compression is triggered for a file then all of the blocks except
52
the last are compressed. Compression will result in a physical block
53
that holds data for more than one logical block. Each logical block now
54
points to the start of compressed data in a physical block with the
55
'phys_pdata' pointer. The 'data_limit' pointer in the physical block is
56
where the compression logic stopped storing data (as stream data
57
compressors are allowed to do). The data for the logical block may span
58
to the next physical block. Once physical blocks are compressed, they are
59
chained together using the 'link' field.
61
The 'f->phys_curr' points to the block being filled by compression, with
62
the 'f->wt.ptr' pointing to the last byte filled in the block. These are
63
used during subsequent compression when the last logical block of the
64
file fills the physical block.
68
During reading the clist, if the logical block points to an uncompressed
69
physical block, then 'memfile_get_pdata' simply sets the 'pdata' and the
70
'pdata_end' pointers. If the logical block was compressed, then it may
71
still be resident in a cache of decompression buffers. The number of these
72
decompression buffers is not critical -- even one is enough, but having
73
more may prevent decompressing blocks more than once (a cache_miss). The
74
number of decompression buffers, called "raw" buffers, that are attempted
75
to allocate can be changed with the GET_NUM_RAW_BUFFERS macro, but no
76
error occurs if less than that number can be allocated.
78
If the logical block still resides in a decompression cache buffer, then
79
the 'raw_block' will identify the block. If the data for a logical block
80
only exists in compressed form, then the "tail" of the list of decompression
81
buffers is re-used, marking the 'raw_block' of the logical block that was
82
previously associated with this data to NULL.
84
Whichever raw decompression buffer is accessed is moved to the head of the
85
decompression buffer list in order to keep the tail of the list as the
86
"least recently used".
88
There are some DEBUG global static variables used to count the number of
89
cache hits "tot_cache_hits" and the number of times a logical block is
90
decompressed "tot_cache_miss". Note that the actual number of cache miss
91
events is 'f->log_length/MEMFILE_DATA_SIZE - tot_cache_miss' since we
92
assume that every logical block must be decmpressed at least once.
94
Empirical results so far indicate that if one cache raw buffer for every
95
32 logical blocks, then the hit/miss ratio exceeds 99%. Of course, the
96
number of raw buffers should be more than 1 if possible, and in many
97
implementations (single threaded), the memory usage does not increase
98
during the page output step so almost all of memory can be used for
99
these raw buffers to prevent the likelihood of a cache miss.
101
Of course, this is dependent on reasonably efficient clist blocking
102
during writing which is dependent on the data and on the BufferSpace
103
value which determines the number of clist band data buffers available.
104
Empirical testing shows that the overall efficiency is best if the
105
BufferSpace value is 1,000,000 (as in the original Ghostscript source).
106
[Note: I expected to be able to use smaller buffer sizes for some cases,
107
but this resulted in a high level of thrashing...RJJ]
111
The most serious limitation is caused by the way 'memfile_fwrite' decides
112
to free up and re-initialize a file. If memfile_fwrite is called after
113
a seek to any location except the start of the file, then an error is
114
issued since logic is not present to properly free up on a partial file.
115
This is not a problem as used by the 'clist' logic since rewind is used
116
to position to the start of a file when re-using it after an 'erasepage'.
118
Since the 'clist' logic always traverses the clist using fseek's to ever
119
increasing locations, no optimizations of backward seeks was implemented.
120
This would be relatively easy with back chain links or bi-directional
121
"X-OR" pointer information to link the logical block chain. The rewind
122
function is optimal and moves directly to the start of the file.
124
********************************************************************************/
127
The need to compress should be conditional on the amount of available
128
memory, but we don't have a way to communicate this to these routines.
129
Instead, we simply start compressing when we've allocated more than
130
COMPRESSION_THRESHOLD amount of data. The threshold should be at
131
least as large as the fixed overhead of the compressor plus the
132
decompressor, plus the expected compressed size of a block that size.
134
static const long COMPRESSION_THRESHOLD = 500000000; /* 0.5 Gb for host machines */
136
#define NEED_TO_COMPRESS(f)\
137
((f)->ok_to_compress && (f)->total_space > COMPRESSION_THRESHOLD)
139
/* FOR NOW ALLOCATE 1 raw buffer for every 32 blocks (at least 8, no more than 64) */
140
#define GET_NUM_RAW_BUFFERS( f ) \
141
min(64, max(f->log_length/MEMFILE_DATA_SIZE/32, 8))
143
#define MALLOC(f, siz, cname)\
144
(void *)gs_alloc_bytes((f)->data_memory, siz, cname)
145
#define FREE(f, obj, cname)\
146
(gs_free_object((f)->data_memory, obj, cname),\
147
(f)->total_space -= sizeof(*(obj)))
149
/* Structure descriptor for GC */
150
private_st_MEMFILE();
152
/* forward references */
153
static void memfile_free_mem(MEMFILE * f);
154
static int memfile_init_empty(MEMFILE * f);
155
static int memfile_set_memory_warning(clist_file_ptr cf, int bytes_left);
156
static int memfile_fclose(clist_file_ptr cf, const char *fname, bool delete);
157
static int memfile_get_pdata(MEMFILE * f);
159
/************************************************/
160
/* #define DEBUG /- force statistics -/ */
161
/************************************************/
171
The following pointers are here only for helping with a dumb debugger
172
that can't inspect local variables!
174
byte *decomp_wt_ptr0, *decomp_wt_limit0;
175
const byte *decomp_rd_ptr0, *decomp_rd_limit0;
176
byte *decomp_wt_ptr1, *decomp_wt_limit1;
177
const byte *decomp_rd_ptr1, *decomp_rd_limit1;
181
/* ----------------------------- Memory Allocation --------------------- */
182
static void * /* allocated memory's address, 0 if failure */
184
MEMFILE *f, /* file to allocate mem to */
185
int sizeofBlock, /* size of block to allocate */
186
int *return_code, /* RET 0 ok, -ve GS-style error, or +1 if OK but low memory */
187
const char *allocName, /* name to allocate by */
188
const char *errorMessage /* error message to print */
191
int code = 0; /* assume success */
192
void *block = MALLOC(f, sizeofBlock, allocName);
195
/* Try to recover block from reserve */
196
if (sizeofBlock == sizeof(LOG_MEMFILE_BLK)) {
197
if (f->reserveLogBlockCount > 0) {
198
block = f->reserveLogBlockChain;
199
f->reserveLogBlockChain = f->reserveLogBlockChain->link;
200
--f->reserveLogBlockCount;
202
} else if (sizeofBlock == sizeof(PHYS_MEMFILE_BLK) ||
203
sizeofBlock == sizeof(RAW_BUFFER)
205
if (f->reservePhysBlockCount > 0) {
206
block = f->reservePhysBlockChain;
207
f->reservePhysBlockChain = f->reservePhysBlockChain->link;
208
--f->reservePhysBlockCount;
212
code = 1; /* successful, but allocated from reserve */
215
f->total_space += sizeofBlock;
217
code = gs_note_error(gs_error_VMerror);
222
/* ---------------- Open/close/unlink ---------------- */
225
memfile_fopen(char fname[gp_file_name_sizeof], const char *fmode,
226
clist_file_ptr /*MEMFILE * */ * pf,
227
gs_memory_t *mem, gs_memory_t *data_mem, bool ok_to_compress)
232
*pf = NULL; /* in case we have an error */
234
/* fname[0] == 0 if this is not reopening */
235
/* memfile file names begin with a flag byte == 0xff */
236
if (fname[0] == '\377' || fmode[0] == 'r') {
237
MEMFILE *base_f = NULL;
239
/* reopening an existing file. */
240
code = sscanf(fname+1, "0x%x", &base_f);
242
gs_note_error(gs_error_ioerror);
245
if (fmode[0] == 'w') {
246
/* Reopen an existing file for 'write' */
247
/* Check first to make sure that we have exclusive access */
248
if (base_f->openlist != NULL) {
249
code = gs_note_error(gs_error_ioerror);
252
f = base_f; /* use the file */
255
/* Reopen an existing file for 'read' */
256
/* We need to 'clone' this memfile so that each reader instance */
257
/* will be able to maintain it's own 'state' */
258
f = gs_alloc_struct(mem, MEMFILE, &st_MEMFILE,
259
"memfile_fopen_instance(MEMFILE)");
261
eprintf1("memfile_open_scratch(%s): gs_alloc_struct failed\n", fname);
262
code = gs_note_error(gs_error_VMerror);
265
memcpy(f, base_f, sizeof(MEMFILE));
267
f->data_memory = data_mem;
268
f->compress_state = 0; /* Not used by reader instance */
269
f->decompress_state = 0; /* make clean for GC, or alloc'n failure */
270
f->reservePhysBlockChain = NULL;
271
f->reservePhysBlockCount = 0;
272
f->reserveLogBlockChain = NULL;
273
f->reserveLogBlockCount = 0;
274
f->openlist = base_f->openlist;
275
base_f->openlist = f; /* link this one in to the base memfile */
276
f->base_memfile = base_f;
281
if (f->log_head->phys_blk->data_limit != NULL) {
282
/* The file is compressed, so we need to copy the logical block */
283
/* list so that it is unique to this instance, and initialize */
284
/* the decompressor. */
285
LOG_MEMFILE_BLK *log_block, *new_log_block;
287
int num_log_blocks = (f->log_length + MEMFILE_DATA_SIZE - 1) / MEMFILE_DATA_SIZE;
288
const stream_state *decompress_proto = clist_decompressor_state(NULL);
289
const stream_template *decompress_template = decompress_proto->template;
291
new_log_block = MALLOC(f, num_log_blocks * sizeof(LOG_MEMFILE_BLK), "memfile_fopen" );
292
if (new_log_block == NULL)
293
code = gs_note_error(gs_error_VMerror);
295
/* copy the logical blocks to the new list just allocated */
296
for (log_block=f->log_head, i=0; log_block != NULL; log_block=log_block->link, i++) {
297
new_log_block[i].phys_blk = log_block->phys_blk;
298
new_log_block[i].phys_pdata = log_block->phys_pdata;
299
new_log_block[i].raw_block = NULL;
300
new_log_block[i].link = log_block->link == NULL ? NULL : new_log_block + i + 1;
302
f->log_head = new_log_block;
304
/* NB: don't need compress_state for reading */
305
f->decompress_state =
306
gs_alloc_struct(mem, stream_state, decompress_template->stype,
307
"memfile_open_scratch(decompress_state)");
308
if (f->decompress_state == 0) {
309
eprintf1("memfile_open_scratch(%s): gs_alloc_struct failed\n", fname);
310
code = gs_note_error(gs_error_VMerror);
313
memcpy(f->decompress_state, decompress_proto,
314
gs_struct_type_size(decompress_template->stype));
315
f->decompress_state->memory = mem;
316
if (decompress_template->set_defaults)
317
(*decompress_template->set_defaults) (f->decompress_state);
319
f->log_curr_blk = f->log_head;
320
memfile_get_pdata(f); /* set up the initial block */
325
fname[0] = 0; /* no file name yet */
326
f = gs_alloc_struct(mem, MEMFILE, &st_MEMFILE,
327
"memfile_open_scratch(MEMFILE)");
329
eprintf1("memfile_open_scratch(%s): gs_alloc_struct failed\n", fname);
330
code = gs_note_error(gs_error_VMerror);
334
f->data_memory = data_mem;
335
/* init an empty file, BEFORE allocating de/compress state */
336
f->compress_state = 0; /* make clean for GC, or alloc'n failure */
337
f->decompress_state = 0;
339
f->base_memfile = NULL;
341
f->reservePhysBlockChain = NULL;
342
f->reservePhysBlockCount = 0;
343
f->reserveLogBlockChain = NULL;
344
f->reserveLogBlockCount = 0;
345
/* init an empty file */
346
if ((code = memfile_init_empty(f)) < 0)
348
if ((code = memfile_set_memory_warning(f, 0)) < 0)
351
* Disregard the ok_to_compress flag, since the size threshold gives us
352
* a much better criterion for deciding when compression is appropriate.
354
f->ok_to_compress = /*ok_to_compress */ true;
355
f->compress_state = 0; /* make clean for GC */
356
f->decompress_state = 0;
357
if (f->ok_to_compress) {
358
const stream_state *compress_proto = clist_compressor_state(NULL);
359
const stream_state *decompress_proto = clist_decompressor_state(NULL);
360
const stream_template *compress_template = compress_proto->template;
361
const stream_template *decompress_template = decompress_proto->template;
364
gs_alloc_struct(mem, stream_state, compress_template->stype,
365
"memfile_open_scratch(compress_state)");
366
f->decompress_state =
367
gs_alloc_struct(mem, stream_state, decompress_template->stype,
368
"memfile_open_scratch(decompress_state)");
369
if (f->compress_state == 0 || f->decompress_state == 0) {
370
eprintf1("memfile_open_scratch(%s): gs_alloc_struct failed\n", fname);
371
code = gs_note_error(gs_error_VMerror);
374
memcpy(f->compress_state, compress_proto,
375
gs_struct_type_size(compress_template->stype));
376
f->compress_state->memory = mem;
377
memcpy(f->decompress_state, decompress_proto,
378
gs_struct_type_size(decompress_template->stype));
379
f->decompress_state->memory = mem;
380
if (compress_template->set_defaults)
381
(*compress_template->set_defaults) (f->compress_state);
382
if (decompress_template->set_defaults)
383
(*decompress_template->set_defaults) (f->decompress_state);
387
/* Return the address of this memfile as a string for use in future clist_fopen calls */
388
fname[0] = 0xff; /* a flag that this is a memfile name */
389
sprintf(fname+1, "0x%0x", f);
400
/* return failure, clean up memory before leaving */
402
memfile_fclose((clist_file_ptr)f, fname, true);
411
memfile_fclose(clist_file_ptr cf, const char *fname, bool delete)
413
MEMFILE *const f = (MEMFILE *)cf;
416
if (f->base_memfile) {
419
/* Here we need to delete this instance from the 'openlist' */
420
/* in case this file was opened for 'read' on a previously */
421
/* written file (base_memfile != NULL) */
422
for (prev_f = f->base_memfile; prev_f != NULL; prev_f = prev_f->openlist)
423
if (prev_f->openlist == f)
425
if (prev_f == NULL) {
426
eprintf1("Could not find 0x%x on memfile openlist\n", ((unsigned int)f));
427
return_error(gs_error_invalidfileaccess);
429
prev_f->openlist = f->openlist; /* link around the one being fclosed */
430
/* Now delete this MEMFILE reader instance */
431
/* NB: we don't delete 'base' instances until we delete */
432
/* If the file is compressed, free the logical blocks, but not */
433
/* the phys_blk info (that is still used by the base memfile */
434
if (f->log_head->phys_blk->data_limit != NULL) {
435
LOG_MEMFILE_BLK *tmpbp, *bp = f->log_head;
439
FREE(f, bp, "memfile_free_mem(log_blk)");
444
/* Free any internal compressor state. */
445
if (f->compressor_initialized) {
446
if (f->decompress_state->template->release != 0)
447
(*f->decompress_state->template->release) (f->decompress_state);
448
if (f->compress_state->template->release != 0)
449
(*f->compress_state->template->release) (f->compress_state);
450
f->compressor_initialized = false;
452
/* free the raw buffers */
453
while (f->raw_head != NULL) {
454
RAW_BUFFER *tmpraw = f->raw_head->fwd;
456
FREE(f, f->raw_head, "memfile_free_mem(raw)");
457
f->raw_head = tmpraw;
460
/* deallocate the memfile object proper */
461
gs_free_object(f->memory, f, "memfile_close_and_unlink(MEMFILE)");
466
/* If there are open read memfile structures, set them so that */
467
/* future accesses will return errors rather than causing SEGV */
469
/* TODO: do the cleanup rather than just giving an error */
470
eprintf1("Attempt to delete a memfile still open for read: 0x%0x\n", ((unsigned int)f));
471
return_error(gs_error_invalidfileaccess);
475
/* Free reserve blocks; don't do it in memfile_free_mem because */
476
/* that routine gets called to reinit file */
477
while (f->reserveLogBlockChain != NULL) {
478
LOG_MEMFILE_BLK *block = f->reserveLogBlockChain;
480
f->reserveLogBlockChain = block->link;
481
FREE(f, block, "memfile_set_block_size");
483
while (f->reservePhysBlockChain != NULL) {
484
PHYS_MEMFILE_BLK *block = f->reservePhysBlockChain;
486
f->reservePhysBlockChain = block->link;
487
FREE(f, block, "memfile_set_block_size");
490
/* deallocate de/compress state */
491
gs_free_object(f->memory, f->decompress_state,
492
"memfile_close_and_unlink(decompress_state)");
493
gs_free_object(f->memory, f->compress_state,
494
"memfile_close_and_unlink(compress_state)");
496
/* deallocate the memfile object proper */
497
gs_free_object(f->memory, f, "memfile_close_and_unlink(MEMFILE)");
502
memfile_unlink(const char *fname)
507
/* memfile file names begin with a flag byte == 0xff */
508
if (fname[0] == '\377' && (code = sscanf(fname+1, "0x%x", &f) == 1)) {
509
return memfile_fclose((clist_file_ptr)f, fname, true);
511
return_error(gs_error_invalidfileaccess);
514
/* ---------------- Writing ---------------- */
516
/* Pre-alloc enough reserve mem blox to guarantee a write of N bytes will succeed */
517
static int /* returns 0 ok, gs_error_VMerror if insufficient */
518
memfile_set_memory_warning(clist_file_ptr cf, int bytes_left)
520
MEMFILE *const f = (MEMFILE *)cf;
523
* Determine req'd memory block count from bytes_left.
524
* Allocate enough phys & log blocks to hold bytes_left
525
* + 1 phys blk for compress_log_blk + 1 phys blk for decompress.
528
(bytes_left + MEMFILE_DATA_SIZE - 1) / MEMFILE_DATA_SIZE;
529
int physNeeded = logNeeded;
533
if (f->raw_head == NULL)
534
++physNeeded; /* have yet to allocate read buffers */
536
/* Allocate or free memory depending on need */
537
while (logNeeded > f->reserveLogBlockCount) {
538
LOG_MEMFILE_BLK *block =
539
MALLOC( f, sizeof(LOG_MEMFILE_BLK), "memfile_set_block_size" );
542
code = gs_note_error(gs_error_VMerror);
545
block->link = f->reserveLogBlockChain;
546
f->reserveLogBlockChain = block;
547
++f->reserveLogBlockCount;
549
while (logNeeded < f->reserveLogBlockCount) {
550
LOG_MEMFILE_BLK *block = f->reserveLogBlockChain;
552
f->reserveLogBlockChain = block->link;
553
FREE(f, block, "memfile_set_block_size");
554
--f->reserveLogBlockCount;
556
while (physNeeded > f->reservePhysBlockCount) {
557
PHYS_MEMFILE_BLK *block =
559
max( sizeof(PHYS_MEMFILE_BLK), sizeof(RAW_BUFFER) ),
560
"memfile_set_block_size");
563
code = gs_note_error(gs_error_VMerror);
566
block->link = f->reservePhysBlockChain;
567
f->reservePhysBlockChain = block;
568
++f->reservePhysBlockCount;
570
while (physNeeded < f->reservePhysBlockCount) {
571
PHYS_MEMFILE_BLK *block = f->reservePhysBlockChain;
573
f->reservePhysBlockChain = block->link;
574
FREE(f, block, "memfile_set_block_size");
575
--f->reservePhysBlockCount;
577
f->error_code = 0; /* memfile_set_block_size is how user resets this */
583
compress_log_blk(MEMFILE * f, LOG_MEMFILE_BLK * bp)
586
int ecode = 0; /* accumulate low-memory warnings */
588
long compressed_size;
590
PHYS_MEMFILE_BLK *newphys;
592
/* compress this block */
593
f->rd.ptr = (const byte *)(bp->phys_blk->data) - 1;
594
f->rd.limit = f->rd.ptr + MEMFILE_DATA_SIZE;
596
bp->phys_blk = f->phys_curr;
597
bp->phys_pdata = (char *)(f->wt.ptr) + 1;
598
if (f->compress_state->template->reinit != 0)
599
(*f->compress_state->template->reinit)(f->compress_state);
602
start_ptr = f->wt.ptr;
603
status = (*f->compress_state->template->process)(f->compress_state,
604
&(f->rd), &(f->wt), true);
605
bp->phys_blk->data_limit = (char *)(f->wt.ptr);
607
if (status == 1) { /* More output space needed (see strimpl.h) */
608
/* allocate another physical block, then compress remainder */
609
compressed_size = f->wt.limit - start_ptr;
611
allocateWithReserve(f, sizeof(*newphys), &code, "memfile newphys",
612
"compress_log_blk : MALLOC for 'newphys' failed\n");
615
ecode |= code; /* accumulate any low-memory warnings */
616
newphys->link = NULL;
617
bp->phys_blk->link = newphys;
618
f->phys_curr = newphys;
619
f->wt.ptr = (byte *) (newphys->data) - 1;
620
f->wt.limit = f->wt.ptr + MEMFILE_DATA_SIZE;
622
start_ptr = f->wt.ptr;
624
(*f->compress_state->template->process)(f->compress_state,
625
&(f->rd), &(f->wt), true);
628
* You'd think the above line is a bug, but in real life 1 src
629
* block never ends up getting split across 3 dest blocks.
631
/* CHANGE memfile_set_memory_warning if this assumption changes. */
632
eprintf("Compression required more than one full block!\n");
633
return_error(gs_error_Fatal);
635
newphys->data_limit = (char *)(f->wt.ptr);
637
compressed_size += f->wt.ptr - start_ptr;
638
if (compressed_size > MEMFILE_DATA_SIZE) {
639
eprintf2("\nCompression didn't - raw=%d, compressed=%ld\n",
640
MEMFILE_DATA_SIZE, compressed_size);
643
tot_compressed += compressed_size;
645
return (status < 0 ? gs_note_error(gs_error_ioerror) : ecode);
646
} /* end "compress_log_blk()" */
648
/* Internal (private) routine to handle end of logical block */
649
static int /* ret 0 ok, -ve error, or +ve low-memory warning */
650
memfile_next_blk(MEMFILE * f)
652
LOG_MEMFILE_BLK *bp = f->log_curr_blk;
653
LOG_MEMFILE_BLK *newbp;
654
PHYS_MEMFILE_BLK *newphys, *oldphys;
655
int ecode = 0; /* accumulate low-memory warnings */
658
if (f->phys_curr == NULL) { /* means NOT compressing */
659
/* allocate a new block */
661
allocateWithReserve(f, sizeof(*newphys), &code, "memfile newphys",
662
"memfile_next_blk: MALLOC 1 for 'newphys' failed\n");
665
ecode |= code; /* accumulate low-mem warnings */
666
newphys->link = NULL;
667
newphys->data_limit = NULL; /* raw */
670
allocateWithReserve(f, sizeof(*newbp), &code, "memfile newbp",
671
"memfile_next_blk: MALLOC 1 for 'newbp' failed\n");
673
FREE(f, newphys, "memfile newphys");
676
ecode |= code; /* accumulate low-mem warnings */
679
newbp->raw_block = NULL;
680
f->log_curr_blk = newbp;
682
/* check if need to start compressing */
683
if (NEED_TO_COMPRESS(f)) {
684
if_debug0(':', "[:]Beginning compression\n");
685
/* compress the entire file up to this point */
686
if (!f->compressor_initialized) {
689
if (f->compress_state->template->init != 0)
690
code = (*f->compress_state->template->init) (f->compress_state);
692
return_error(gs_error_VMerror); /****** BOGUS ******/
693
f->compressor_initialized = true;
695
/* Write into the new physical block we just allocated, */
696
/* replace it after the loop (after some blocks are freed) */
697
f->phys_curr = newphys;
698
f->wt.ptr = (byte *) (newphys->data) - 1;
699
f->wt.limit = f->wt.ptr + MEMFILE_DATA_SIZE;
701
while (bp != newbp) { /* don't compress last block */
704
oldphys = bp->phys_blk;
705
if ((code = compress_log_blk(f, bp)) < 0)
708
FREE(f, oldphys, "memfile_next_blk(oldphys)");
710
} /* end while( ) compress loop */
711
/* Allocate a physical block for this (last) logical block */
713
allocateWithReserve(f, sizeof(*newphys), &code,
715
"memfile_next_blk: MALLOC 2 for 'newphys' failed\n");
718
ecode |= code; /* accumulate low-mem warnings */
719
newphys->link = NULL;
720
newphys->data_limit = NULL; /* raw */
722
} /* end convert file to compressed */
723
newbp->phys_blk = newphys;
724
f->pdata = newphys->data;
725
f->pdata_end = newphys->data + MEMFILE_DATA_SIZE;
726
} /* end if NOT compressing */
727
/* File IS being compressed */
731
oldphys = bp->phys_blk; /* save raw phys block ID */
732
/* compresses bp on phys list */
733
if ((code = compress_log_blk(f, bp)) < 0)
737
allocateWithReserve(f, sizeof(*newbp), &code, "memfile newbp",
738
"memfile_next_blk: MALLOC 2 for 'newbp' failed\n");
744
newbp->raw_block = NULL;
745
/* Re-use the raw phys block for this new logical blk */
746
newbp->phys_blk = oldphys;
747
f->pdata = oldphys->data;
748
f->pdata_end = f->pdata + MEMFILE_DATA_SIZE;
749
f->log_curr_blk = newbp;
750
} /* end else (when we are compressing) */
755
static int /* returns # of chars actually written */
756
memfile_fwrite_chars(const void *data, uint len, clist_file_ptr cf)
758
const char *str = (const char *)data;
759
MEMFILE *f = (MEMFILE *) cf;
763
/* check if we are writing to the start of the file. If so, then */
764
/* free the file memory and re-initialize it (frees memory) */
765
if (f->log_curr_pos == 0) {
769
if ((code = memfile_init_empty(f)) < 0) {
770
f->error_code = code;
774
if (f->log_curr_blk->link != 0) {
775
eprintf(" Write file truncate -- need to free physical blocks.\n");
778
uint move_count = f->pdata_end - f->pdata;
780
if (move_count == 0) {
781
if ((ecode = memfile_next_blk(f)) != 0) {
782
f->error_code = ecode;
787
if (move_count > count)
789
memmove(f->pdata, str, move_count);
790
f->pdata += move_count;
795
f->log_curr_pos += len;
796
f->log_length = f->log_curr_pos; /* truncate length to here */
804
/* Internal routine to set the f->pdata and f->pdata_end pointers */
805
/* for the current logical block f->log_curr_blk */
807
/* If data only exists in compressed form, allocate a raw buffer */
808
/* and decompress it. */
812
memfile_get_pdata(MEMFILE * f)
814
int code, i, num_raw_buffers, status;
815
LOG_MEMFILE_BLK *bp = f->log_curr_blk;
817
if (bp->phys_blk->data_limit == NULL) {
818
/* Not compressed, return this data pointer */
819
f->pdata = (bp->phys_blk)->data;
820
i = f->log_curr_pos % MEMFILE_DATA_SIZE; /* pos within block */
821
i = f->log_curr_pos - i; /* base of block */
822
if (i + MEMFILE_DATA_SIZE > f->log_length)
823
f->pdata_end = f->pdata + f->log_length - i;
825
f->pdata_end = f->pdata + MEMFILE_DATA_SIZE;
828
/* data was compressed */
829
if (f->raw_head == NULL) {
831
/* need to allocate the raw buffer pool */
832
num_raw_buffers = GET_NUM_RAW_BUFFERS(f);
833
if (f->reservePhysBlockCount) {
834
/* HACK: allocate reserve block that's been reserved for
835
* decompression. This buffer's block was pre-allocated to make
836
* sure we won't come up short here. Take from chain instead of
837
* allocateWithReserve() since this buf would just be wasted if
838
* allowed to remain preallocated. */
839
f->raw_head = (RAW_BUFFER *)f->reservePhysBlockChain;
840
f->reservePhysBlockChain = f->reservePhysBlockChain->link;
841
--f->reservePhysBlockCount;
844
allocateWithReserve(f, sizeof(*f->raw_head), &code,
845
"memfile raw buffer",
846
"memfile_get_pdata: MALLOC for 'raw_head' failed\n");
850
f->raw_head->back = NULL;
851
f->raw_tail = f->raw_head;
852
f->raw_tail->log_blk = NULL;
853
for (i = 0; i < num_raw_buffers; i++) {
854
f->raw_tail->fwd = (RAW_BUFFER *) MALLOC(f, sizeof(RAW_BUFFER),
855
"memfile raw buffer");
856
/* if MALLOC fails, then just stop allocating */
857
if (!f->raw_tail->fwd)
859
f->total_space += sizeof(RAW_BUFFER);
860
f->raw_tail->fwd->back = f->raw_tail;
861
f->raw_tail = f->raw_tail->fwd;
862
f->raw_tail->log_blk = NULL;
864
f->raw_tail->fwd = NULL;
865
num_raw_buffers = i + 1; /* if MALLOC failed, then OK */
866
if_debug1(':', "[:]Number of raw buffers allocated=%d\n",
868
if (f->decompress_state->template->init != 0)
869
code = (*f->decompress_state->template->init)
870
(f->decompress_state);
872
return_error(gs_error_VMerror);
874
} /* end allocating the raw buffer pool (first time only) */
875
if (bp->raw_block == NULL) {
877
tot_cache_miss++; /* count every decompress */
879
/* find a raw buffer and decompress */
880
if (f->raw_tail->log_blk != NULL) {
881
/* This block was in use, grab it */
885
f->raw_tail->log_blk->raw_block = NULL; /* data no longer here */
886
f->raw_tail->log_blk = NULL;
888
/* Use the last raw block in the chain (the oldest) */
889
f->raw_tail->back->fwd = NULL; /* disconnect from tail */
890
f->raw_tail->fwd = f->raw_head; /* new head */
891
f->raw_head->back = f->raw_tail;
892
f->raw_tail = f->raw_tail->back;
893
f->raw_head = f->raw_head->back;
894
f->raw_head->back = NULL;
895
f->raw_head->log_blk = bp;
897
/* Decompress the data into this raw block */
898
/* Initialize the decompressor */
899
if (f->decompress_state->template->reinit != 0)
900
(*f->decompress_state->template->reinit) (f->decompress_state);
901
/* Set pointers and call the decompress routine */
902
f->wt.ptr = (byte *) (f->raw_head->data) - 1;
903
f->wt.limit = f->wt.ptr + MEMFILE_DATA_SIZE;
904
f->rd.ptr = (const byte *)(bp->phys_pdata) - 1;
905
f->rd.limit = (const byte *)bp->phys_blk->data_limit;
907
decomp_wt_ptr0 = f->wt.ptr;
908
decomp_wt_limit0 = f->wt.limit;
909
decomp_rd_ptr0 = f->rd.ptr;
910
decomp_rd_limit0 = f->rd.limit;
912
status = (*f->decompress_state->template->process)
913
(f->decompress_state, &(f->rd), &(f->wt), true);
914
if (status == 0) { /* More input data needed */
915
/* switch to next block and continue decompress */
916
int back_up = 0; /* adjust pointer backwards */
918
if (f->rd.ptr != f->rd.limit) {
919
/* transfer remainder bytes from the previous block */
920
back_up = f->rd.limit - f->rd.ptr;
921
for (i = 0; i < back_up; i++)
922
*(bp->phys_blk->link->data - back_up + i) = *++f->rd.ptr;
924
f->rd.ptr = (const byte *)bp->phys_blk->link->data - back_up - 1;
925
f->rd.limit = (const byte *)bp->phys_blk->link->data_limit;
927
decomp_wt_ptr1 = f->wt.ptr;
928
decomp_wt_limit1 = f->wt.limit;
929
decomp_rd_ptr1 = f->rd.ptr;
930
decomp_rd_limit1 = f->rd.limit;
932
status = (*f->decompress_state->template->process)
933
(f->decompress_state, &(f->rd), &(f->wt), true);
935
eprintf("Decompression required more than one full block!\n");
936
return_error(gs_error_Fatal);
939
bp->raw_block = f->raw_head; /* point to raw block */
941
/* end if( raw_block == NULL ) meaning need to decompress data */
943
/* data exists in the raw data cache, if not raw_head, move it */
944
if (bp->raw_block != f->raw_head) {
945
/* move to raw_head */
946
/* prev.fwd = this.fwd */
947
bp->raw_block->back->fwd = bp->raw_block->fwd;
948
if (bp->raw_block->fwd != NULL)
949
/* next.back = this.back */
950
bp->raw_block->fwd->back = bp->raw_block->back;
952
f->raw_tail = bp->raw_block->back; /* tail = prev */
953
f->raw_head->back = bp->raw_block; /* head.back = this */
954
bp->raw_block->fwd = f->raw_head; /* this.fwd = orig head */
955
f->raw_head = bp->raw_block; /* head = this */
956
f->raw_head->back = NULL; /* this.back = NULL */
958
tot_cache_hits++; /* counting here prevents repeats since */
959
/* won't count if already at head */
963
f->pdata = bp->raw_block->data;
964
f->pdata_end = f->pdata + MEMFILE_DATA_SIZE;
965
/* NOTE: last block is never compressed, so a compressed block */
966
/* is always full size. */
967
} /* end else (when data was compressed) */
972
/* ---------------- Reading ---------------- */
975
memfile_fread_chars(void *data, uint len, clist_file_ptr cf)
977
char *str = (char *)data;
978
MEMFILE *f = (MEMFILE *) cf;
979
uint count = len, num_read, move_count;
981
num_read = f->log_length - f->log_curr_pos;
982
if (count > num_read)
987
f->log_curr_pos++; /* move into next byte */
988
if (f->pdata == f->pdata_end) {
989
f->log_curr_blk = (f->log_curr_blk)->link;
990
memfile_get_pdata(f);
992
move_count = f->pdata_end - f->pdata;
993
if (move_count > count)
995
f->log_curr_pos += move_count - 1; /* new position */
996
memmove(str, f->pdata, move_count);
998
f->pdata += move_count;
1005
/* ---------------- Position/status ---------------- */
1008
memfile_ferror_code(clist_file_ptr cf)
1010
return (((MEMFILE *) cf)->error_code); /* errors stored here */
1014
memfile_ftell(clist_file_ptr cf)
1016
return (((MEMFILE *) cf)->log_curr_pos);
1020
memfile_rewind(clist_file_ptr cf, bool discard_data, const char *ignore_fname)
1022
MEMFILE *f = (MEMFILE *) cf;
1025
memfile_free_mem(f);
1026
/* We have to call memfile_init_empty to preserve invariants. */
1027
memfile_init_empty(f);
1029
f->log_curr_blk = f->log_head;
1030
f->log_curr_pos = 0;
1031
memfile_get_pdata(f);
1036
memfile_fseek(clist_file_ptr cf, int64_t offset, int mode, const char *ignore_fname)
1038
MEMFILE *f = (MEMFILE *) cf;
1039
long i, block_num, new_pos;
1042
case SEEK_SET: /* offset from the beginning of the file */
1046
case SEEK_CUR: /* offset from the current position in the file */
1047
new_pos = offset + f->log_curr_pos;
1050
case SEEK_END: /* offset back from the end of the file */
1051
new_pos = f->log_length - offset;
1057
if (new_pos < 0 || new_pos > f->log_length)
1059
if ((f->pdata == f->pdata_end) && (f->log_curr_blk->link != NULL)) {
1060
/* log_curr_blk is actually one block behind log_curr_pos */
1061
f->log_curr_blk = f->log_curr_blk->link;
1063
block_num = new_pos / MEMFILE_DATA_SIZE;
1064
i = f->log_curr_pos / MEMFILE_DATA_SIZE;
1065
if (block_num < i) { /* if moving backwards, start at beginning */
1066
f->log_curr_blk = f->log_head;
1069
for (; i < block_num; i++) {
1070
f->log_curr_blk = f->log_curr_blk->link;
1072
f->log_curr_pos = new_pos;
1073
memfile_get_pdata(f); /* pointers to start of block */
1074
f->pdata += new_pos - (block_num * MEMFILE_DATA_SIZE);
1076
return 0; /* return "normal" status */
1079
/* ---------------- Internal routines ---------------- */
1082
memfile_free_mem(MEMFILE * f)
1084
LOG_MEMFILE_BLK *bp, *tmpbp;
1087
/* output some diagnostics about the effectiveness */
1088
if (tot_raw > 100) {
1089
if_debug2(':', "[:]tot_raw=%ld, tot_compressed=%ld\n",
1090
tot_raw, tot_compressed);
1092
if (tot_cache_hits != 0) {
1093
if_debug3(':', "[:]Cache hits=%ld, cache misses=%ld, swapouts=%ld\n",
1095
(long)(tot_cache_miss - (f->log_length / MEMFILE_DATA_SIZE)),
1105
/* Free up memory that was allocated for the memfile */
1109
/* Null out phys_blk pointers to compressed data. */
1110
PHYS_MEMFILE_BLK *pphys = bp->phys_blk;
1113
for (tmpbp = bp; tmpbp != NULL; tmpbp = tmpbp->link)
1114
if (tmpbp->phys_blk->data_limit != NULL)
1115
tmpbp->phys_blk = 0;
1117
/* Free the physical blocks that make up the compressed data */
1118
if (pphys->data_limit != NULL) {
1119
/* the data was compressed, free the chain of blocks */
1120
while (pphys != NULL) {
1121
PHYS_MEMFILE_BLK *tmpphys = pphys->link;
1123
FREE(f, pphys, "memfile_free_mem(pphys)");
1128
/* Now free the logical blocks, and any uncompressed physical blocks. */
1129
while (bp != NULL) {
1130
if (bp->phys_blk != NULL) {
1131
FREE(f, bp->phys_blk, "memfile_free_mem(phys_blk)");
1134
FREE(f, bp, "memfile_free_mem(log_blk)");
1140
/* Free any internal compressor state. */
1141
if (f->compressor_initialized) {
1142
if (f->decompress_state->template->release != 0)
1143
(*f->decompress_state->template->release) (f->decompress_state);
1144
if (f->compress_state->template->release != 0)
1145
(*f->compress_state->template->release) (f->compress_state);
1146
f->compressor_initialized = false;
1148
/* free the raw buffers */
1149
while (f->raw_head != NULL) {
1150
RAW_BUFFER *tmpraw = f->raw_head->fwd;
1152
FREE(f, f->raw_head, "memfile_free_mem(raw)");
1153
f->raw_head = tmpraw;
1158
memfile_init_empty(MEMFILE * f)
1160
PHYS_MEMFILE_BLK *pphys;
1161
LOG_MEMFILE_BLK *plog;
1163
/* Zero out key fields so that allocation failure will be unwindable */
1164
f->phys_curr = NULL; /* flag as file not compressed */
1166
f->log_curr_blk = NULL;
1167
f->log_curr_pos = 0;
1170
f->compressor_initialized = false;
1173
/* File empty - get a physical mem block (includes the buffer area) */
1174
pphys = MALLOC(f, sizeof(*pphys), "memfile pphys");
1176
eprintf("memfile_init_empty: MALLOC for 'pphys' failed\n");
1177
return_error(gs_error_VMerror);
1179
f->total_space += sizeof(*pphys);
1180
pphys->data_limit = NULL; /* raw data for now */
1182
/* Get logical mem block to go with physical one */
1183
plog = (LOG_MEMFILE_BLK *)MALLOC( f, sizeof(*plog), "memfile_init_empty" );
1185
FREE(f, pphys, "memfile_init_empty");
1186
eprintf("memfile_init_empty: MALLOC for log_curr_blk failed\n");
1187
return_error(gs_error_VMerror);
1189
f->total_space += sizeof(*plog);
1190
f->log_head = f->log_curr_blk = plog;
1191
f->log_curr_blk->link = NULL;
1192
f->log_curr_blk->phys_blk = pphys;
1193
f->log_curr_blk->phys_pdata = NULL;
1194
f->log_curr_blk->raw_block = NULL;
1196
f->pdata = pphys->data;
1197
f->pdata_end = f->pdata + MEMFILE_DATA_SIZE;
1204
clist_io_procs_t clist_io_procs_memory = {
1208
memfile_fwrite_chars,
1209
memfile_fread_chars,
1210
memfile_set_memory_warning,
1211
memfile_ferror_code,
1217
init_proc(gs_gxclmem_init);
1219
gs_gxclmem_init(gs_memory_t *mem)
1221
clist_io_procs_memory_global = &clist_io_procs_memory;
added
removed
src/gxclmem.c
