← Back to branch summary

~ubuntu-branches/ubuntu/wily/dovecot/wily

« back to all changes in this revision

Viewing changes to src/lib-compression/istream-zlib.c

  • Committer: Package Import Robot
  • Author(s): Jaldhar H. Vyas
  • Date: 2013-09-09 00:57:32 UTC
  • mfrom: (1.13.11)
  • mto: (4.8.5 experimental) (1.16.1)
  • mto: This revision was merged to the branch mainline in revision 97.
  • Revision ID: package-import@ubuntu.com-20130909005732-dn1eell8srqbhh0e
Tags: upstream-2.2.5
Import upstream version 2.2.5

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/lib-compression/istream-zlib.c
 
1
/* Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file */
 
2
 
 
3
#include "lib.h"
 
4
 
 
5
#ifdef HAVE_ZLIB
 
6
 
 
7
#include "crc32.h"
 
8
#include "istream-private.h"
 
9
#include "istream-zlib.h"
 
10
#include <zlib.h>
 
11
 
 
12
#define CHUNK_SIZE (1024*64)
 
13
 
 
14
#define GZ_HEADER_MIN_SIZE 10
 
15
#define GZ_TRAILER_SIZE 8
 
16
 
 
17
#define GZ_MAGIC1       0x1f
 
18
#define GZ_MAGIC2       0x8b
 
19
#define GZ_FLAG_FHCRC   0x02
 
20
#define GZ_FLAG_FEXTRA  0x04
 
21
#define GZ_FLAG_FNAME   0x08
 
22
#define GZ_FLAG_FCOMMENT 0x10
 
23
 
 
24
struct zlib_istream {
 
25
        struct istream_private istream;
 
26
 
 
27
        z_stream zs;
 
28
        uoff_t eof_offset, stream_size;
 
29
        size_t prev_size, high_pos;
 
30
        uint32_t crc32;
 
31
        struct stat last_parent_statbuf;
 
32
 
 
33
        unsigned int gz:1;
 
34
        unsigned int log_errors:1;
 
35
        unsigned int marked:1;
 
36
        unsigned int header_read:1;
 
37
        unsigned int trailer_read:1;
 
38
        unsigned int zs_closed:1;
 
39
};
 
40
 
 
41
static void i_stream_zlib_init(struct zlib_istream *zstream);
 
42
 
 
43
static void i_stream_zlib_close(struct iostream_private *stream,
 
44
                                bool close_parent)
 
45
{
 
46
        struct zlib_istream *zstream = (struct zlib_istream *)stream;
 
47
 
 
48
        if (!zstream->zs_closed) {
 
49
                (void)inflateEnd(&zstream->zs);
 
50
                zstream->zs_closed = TRUE;
 
51
        }
 
52
        if (close_parent)
 
53
                i_stream_close(zstream->istream.parent);
 
54
}
 
55
 
 
56
static void zlib_read_error(struct zlib_istream *zstream, const char *error)
 
57
{
 
58
        i_error("zlib.read(%s): %s at %"PRIuUOFF_T,
 
59
                i_stream_get_name(&zstream->istream.istream), error,
 
60
                zstream->istream.abs_start_offset +
 
61
                zstream->istream.istream.v_offset);
 
62
}
 
63
 
 
64
static int i_stream_zlib_read_header(struct istream_private *stream)
 
65
{
 
66
        struct zlib_istream *zstream = (struct zlib_istream *)stream;
 
67
        const unsigned char *data;
 
68
        size_t size;
 
69
        unsigned int pos, fextra_size;
 
70
        int ret;
 
71
 
 
72
        ret = i_stream_read_data(stream->parent, &data, &size,
 
73
                                 zstream->prev_size);
 
74
        if (size == zstream->prev_size) {
 
75
                if (ret == -1) {
 
76
                        if (zstream->log_errors)
 
77
                                zlib_read_error(zstream, "missing gz header");
 
78
                        stream->istream.stream_errno = EINVAL;
 
79
                }
 
80
                return ret;
 
81
        }
 
82
        zstream->prev_size = size;
 
83
 
 
84
        if (size < GZ_HEADER_MIN_SIZE)
 
85
                return 0;
 
86
        pos = GZ_HEADER_MIN_SIZE;
 
87
 
 
88
        if (data[0] != GZ_MAGIC1 || data[1] != GZ_MAGIC2) {
 
89
                /* missing gzip magic header */
 
90
                if (zstream->log_errors) {
 
91
                        zlib_read_error(zstream, "wrong magic in header "
 
92
                                        "(not gz file?)");
 
93
                }
 
94
                stream->istream.stream_errno = EINVAL;
 
95
                return -1;
 
96
        }
 
97
        if ((data[3] & GZ_FLAG_FEXTRA) != 0) {
 
98
                if (pos + 2 < size)
 
99
                        return 0;
 
100
 
 
101
                fextra_size = data[pos] + (data[pos+1] << 8);
 
102
                pos += 2;
 
103
                if (pos + fextra_size < size)
 
104
                        return 0;
 
105
                pos += fextra_size;
 
106
        }
 
107
        if ((data[3] & GZ_FLAG_FNAME) != 0) {
 
108
                do {
 
109
                        if (pos == size)
 
110
                                return 0;
 
111
                } while (data[pos++] != '\0');
 
112
        }
 
113
        if ((data[3] & GZ_FLAG_FCOMMENT) != 0) {
 
114
                do {
 
115
                        if (pos == size)
 
116
                                return 0;
 
117
                } while (data[pos++] != '\0');
 
118
        }
 
119
        if ((data[3] & GZ_FLAG_FHCRC) != 0) {
 
120
                if (pos + 2 < size)
 
121
                        return 0;
 
122
                pos += 2;
 
123
        }
 
124
        i_stream_skip(stream->parent, pos);
 
125
        zstream->prev_size = 0;
 
126
        return 1;
 
127
}
 
128
 
 
129
static uint32_t data_get_uint32(const unsigned char *data)
 
130
{
 
131
        return data[0] | (data[1] << 8) | (data[2] << 16) |
 
132
                ((uint32_t)data[3] << 24);
 
133
}
 
134
 
 
135
static int i_stream_zlib_read_trailer(struct zlib_istream *zstream)
 
136
{
 
137
        struct istream_private *stream = &zstream->istream;
 
138
        const unsigned char *data;
 
139
        size_t size;
 
140
        int ret;
 
141
 
 
142
        ret = i_stream_read_data(stream->parent, &data, &size,
 
143
                                 GZ_TRAILER_SIZE-1);
 
144
        if (size == zstream->prev_size) {
 
145
                if (ret == -1) {
 
146
                        if (zstream->log_errors)
 
147
                                zlib_read_error(zstream, "missing gz trailer");
 
148
                        stream->istream.stream_errno = EINVAL;
 
149
                }
 
150
                return ret;
 
151
        }
 
152
        zstream->prev_size = size;
 
153
 
 
154
        if (size < GZ_TRAILER_SIZE)
 
155
                return 0;
 
156
 
 
157
        if (data_get_uint32(data) != zstream->crc32) {
 
158
                if (zstream->log_errors) {
 
159
                        zlib_read_error(zstream,
 
160
                                        "gz trailer has wrong CRC value");
 
161
                }
 
162
                stream->istream.stream_errno = EINVAL;
 
163
                return -1;
 
164
        }
 
165
        i_stream_skip(stream->parent, GZ_TRAILER_SIZE);
 
166
        zstream->prev_size = 0;
 
167
        zstream->trailer_read = TRUE;
 
168
        return 1;
 
169
}
 
170
 
 
171
static ssize_t i_stream_zlib_read(struct istream_private *stream)
 
172
{
 
173
        struct zlib_istream *zstream = (struct zlib_istream *)stream;
 
174
        const unsigned char *data;
 
175
        uoff_t high_offset;
 
176
        size_t size, out_size;
 
177
        int ret;
 
178
 
 
179
        high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
 
180
        if (zstream->eof_offset == high_offset) {
 
181
                i_assert(zstream->high_pos == 0 ||
 
182
                         zstream->high_pos == stream->pos);
 
183
                if (!zstream->trailer_read) {
 
184
                        do {
 
185
                                ret = i_stream_zlib_read_trailer(zstream);
 
186
                        } while (ret == 0 && stream->istream.blocking);
 
187
                        if (ret <= 0)
 
188
                                return ret;
 
189
                }
 
190
                if (!zstream->gz || i_stream_is_eof(stream->parent)) {
 
191
                        stream->istream.eof = TRUE;
 
192
                        return -1;
 
193
                }
 
194
                /* gzip file with concatenated content */
 
195
                zstream->eof_offset = (uoff_t)-1;
 
196
                zstream->stream_size = (uoff_t)-1;
 
197
                zstream->header_read = FALSE;
 
198
                zstream->trailer_read = FALSE;
 
199
                zstream->crc32 = 0;
 
200
 
 
201
                (void)inflateEnd(&zstream->zs);
 
202
                i_stream_zlib_init(zstream);
 
203
        }
 
204
 
 
205
        if (!zstream->header_read) {
 
206
                do {
 
207
                        ret = i_stream_zlib_read_header(stream);
 
208
                } while (ret == 0 && stream->istream.blocking);
 
209
                if (ret <= 0)
 
210
                        return ret;
 
211
                zstream->header_read = TRUE;
 
212
        }
 
213
 
 
214
        if (stream->pos < zstream->high_pos) {
 
215
                /* we're here because we seeked back within the read buffer. */
 
216
                ret = zstream->high_pos - stream->pos;
 
217
                stream->pos = zstream->high_pos;
 
218
                zstream->high_pos = 0;
 
219
                if (zstream->trailer_read) {
 
220
                        high_offset = stream->istream.v_offset +
 
221
                                (stream->pos - stream->skip);
 
222
                        i_assert(zstream->eof_offset == high_offset);
 
223
                        stream->istream.eof = TRUE;
 
224
                }
 
225
                return ret;
 
226
        }
 
227
        zstream->high_pos = 0;
 
228
 
 
229
        if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
 
230
                /* try to keep at least CHUNK_SIZE available */
 
231
                if (!zstream->marked && stream->skip > 0) {
 
232
                        /* don't try to keep anything cached if we don't
 
233
                           have a seek mark. */
 
234
                        i_stream_compress(stream);
 
235
                }
 
236
                if (stream->max_buffer_size == 0 ||
 
237
                    stream->buffer_size < stream->max_buffer_size)
 
238
                        i_stream_grow_buffer(stream, CHUNK_SIZE);
 
239
 
 
240
                if (stream->pos == stream->buffer_size) {
 
241
                        if (stream->skip > 0) {
 
242
                                /* lose our buffer cache */
 
243
                                i_stream_compress(stream);
 
244
                        }
 
245
 
 
246
                        if (stream->pos == stream->buffer_size)
 
247
                                return -2; /* buffer full */
 
248
                }
 
249
        }
 
250
 
 
251
        if (i_stream_read_data(stream->parent, &data, &size, 0) < 0) {
 
252
                if (stream->parent->stream_errno != 0) {
 
253
                        stream->istream.stream_errno =
 
254
                                stream->parent->stream_errno;
 
255
                } else {
 
256
                        i_assert(stream->parent->eof);
 
257
                        if (zstream->log_errors)
 
258
                                zlib_read_error(zstream, "unexpected EOF");
 
259
                        stream->istream.stream_errno = EPIPE;
 
260
                }
 
261
                return -1;
 
262
        }
 
263
        if (size == 0) {
 
264
                /* no more input */
 
265
                i_assert(!stream->istream.blocking);
 
266
                return 0;
 
267
        }
 
268
 
 
269
        zstream->zs.next_in = (void *)data;
 
270
        zstream->zs.avail_in = size;
 
271
 
 
272
        out_size = stream->buffer_size - stream->pos;
 
273
        zstream->zs.next_out = stream->w_buffer + stream->pos;
 
274
        zstream->zs.avail_out = out_size;
 
275
        ret = inflate(&zstream->zs, Z_SYNC_FLUSH);
 
276
 
 
277
        out_size -= zstream->zs.avail_out;
 
278
        zstream->crc32 = crc32_data_more(zstream->crc32,
 
279
                                         stream->w_buffer + stream->pos,
 
280
                                         out_size);
 
281
        stream->pos += out_size;
 
282
 
 
283
        i_stream_skip(stream->parent, size - zstream->zs.avail_in);
 
284
 
 
285
        switch (ret) {
 
286
        case Z_OK:
 
287
                break;
 
288
        case Z_NEED_DICT:
 
289
                if (zstream->log_errors)
 
290
                        zlib_read_error(zstream, "can't read file without dict");
 
291
                stream->istream.stream_errno = EINVAL;
 
292
                return -1;
 
293
        case Z_DATA_ERROR:
 
294
                if (zstream->log_errors)
 
295
                        zlib_read_error(zstream, "corrupted data");
 
296
                stream->istream.stream_errno = EINVAL;
 
297
                return -1;
 
298
        case Z_MEM_ERROR:
 
299
                i_fatal_status(FATAL_OUTOFMEM, "zlib.read(%s): Out of memory",
 
300
                               i_stream_get_name(&stream->istream));
 
301
        case Z_STREAM_END:
 
302
                zstream->eof_offset = stream->istream.v_offset +
 
303
                        (stream->pos - stream->skip);
 
304
                zstream->stream_size = zstream->eof_offset;
 
305
                zstream->zs.avail_in = 0;
 
306
 
 
307
                if (!zstream->trailer_read) {
 
308
                        /* try to read and verify the trailer, we might not
 
309
                           be called again. */
 
310
                        if (i_stream_zlib_read_trailer(zstream) < 0)
 
311
                                return -1;
 
312
                }
 
313
                break;
 
314
        default:
 
315
                i_fatal("inflate() failed with %d", ret);
 
316
        }
 
317
        if (out_size == 0) {
 
318
                /* read more input */
 
319
                return i_stream_zlib_read(stream);
 
320
        }
 
321
        return out_size;
 
322
}
 
323
 
 
324
static void i_stream_zlib_init(struct zlib_istream *zstream)
 
325
{
 
326
        int ret;
 
327
 
 
328
        ret = inflateInit2(&zstream->zs, -15);
 
329
        switch (ret) {
 
330
        case Z_OK:
 
331
                break;
 
332
        case Z_MEM_ERROR:
 
333
                i_fatal_status(FATAL_OUTOFMEM, "zlib: Out of memory");
 
334
        case Z_VERSION_ERROR:
 
335
                i_fatal("Wrong zlib library version (broken compilation)");
 
336
        case Z_STREAM_ERROR:
 
337
                i_fatal("zlib: Invalid parameters");
 
338
        default:
 
339
                i_fatal("inflateInit() failed with %d", ret);
 
340
        }
 
341
        zstream->header_read = !zstream->gz;
 
342
        zstream->trailer_read = !zstream->gz;
 
343
}
 
344
 
 
345
static void i_stream_zlib_reset(struct zlib_istream *zstream)
 
346
{
 
347
        struct istream_private *stream = &zstream->istream;
 
348
 
 
349
        i_stream_seek(stream->parent, stream->parent_start_offset);
 
350
        zstream->eof_offset = (uoff_t)-1;
 
351
        zstream->crc32 = 0;
 
352
 
 
353
        zstream->zs.next_in = NULL;
 
354
        zstream->zs.avail_in = 0;
 
355
 
 
356
        stream->parent_expected_offset = stream->parent_start_offset;
 
357
        stream->skip = stream->pos = 0;
 
358
        stream->istream.v_offset = 0;
 
359
        zstream->high_pos = 0;
 
360
        zstream->prev_size = 0;
 
361
 
 
362
        (void)inflateEnd(&zstream->zs);
 
363
        i_stream_zlib_init(zstream);
 
364
}
 
365
 
 
366
static void
 
367
i_stream_zlib_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
 
368
{
 
369
        struct zlib_istream *zstream = (struct zlib_istream *) stream;
 
370
        uoff_t start_offset = stream->istream.v_offset - stream->skip;
 
371
 
 
372
        if (v_offset < start_offset) {
 
373
                /* have to seek backwards */
 
374
                i_stream_zlib_reset(zstream);
 
375
                start_offset = 0;
 
376
        } else if (zstream->high_pos != 0) {
 
377
                stream->pos = zstream->high_pos;
 
378
                zstream->high_pos = 0;
 
379
        }
 
380
 
 
381
        if (v_offset <= start_offset + stream->pos) {
 
382
                /* seeking backwards within what's already cached */
 
383
                stream->skip = v_offset - start_offset;
 
384
                stream->istream.v_offset = v_offset;
 
385
                zstream->high_pos = stream->pos;
 
386
                stream->pos = stream->skip;
 
387
        } else {
 
388
                /* read and cache forward */
 
389
                do {
 
390
                        size_t avail = stream->pos - stream->skip;
 
391
 
 
392
                        if (stream->istream.v_offset + avail >= v_offset) {
 
393
                                i_stream_skip(&stream->istream,
 
394
                                              v_offset -
 
395
                                              stream->istream.v_offset);
 
396
                                break;
 
397
                        }
 
398
 
 
399
                        i_stream_skip(&stream->istream, avail);
 
400
                } while (i_stream_read(&stream->istream) >= 0);
 
401
 
 
402
                if (stream->istream.v_offset != v_offset) {
 
403
                        /* some failure, we've broken it */
 
404
                        if (stream->istream.stream_errno != 0) {
 
405
                                i_error("zlib_istream.seek(%s) failed: %s",
 
406
                                        i_stream_get_name(&stream->istream),
 
407
                                        strerror(stream->istream.stream_errno));
 
408
                                i_stream_close(&stream->istream);
 
409
                        } else {
 
410
                                /* unexpected EOF. allow it since we may just
 
411
                                   want to check if there's anything.. */
 
412
                                i_assert(stream->istream.eof);
 
413
                        }
 
414
                }
 
415
        }
 
416
 
 
417
        if (mark)
 
418
                zstream->marked = TRUE;
 
419
}
 
420
 
 
421
static int
 
422
i_stream_zlib_stat(struct istream_private *stream, bool exact)
 
423
{
 
424
        struct zlib_istream *zstream = (struct zlib_istream *) stream;
 
425
        const struct stat *st;
 
426
        size_t size;
 
427
 
 
428
        if (i_stream_stat(stream->parent, exact, &st) < 0)
 
429
                return -1;
 
430
        stream->statbuf = *st;
 
431
 
 
432
        /* when exact=FALSE always return the parent stat's size, even if we
 
433
           know the exact value. this is necessary because otherwise e.g. mbox
 
434
           code can see two different values and think that a compressed mbox
 
435
           file keeps changing. */
 
436
        if (!exact)
 
437
                return 0;
 
438
 
 
439
        if (zstream->stream_size == (uoff_t)-1) {
 
440
                uoff_t old_offset = stream->istream.v_offset;
 
441
 
 
442
                do {
 
443
                        size = i_stream_get_data_size(&stream->istream);
 
444
                        i_stream_skip(&stream->istream, size);
 
445
                } while (i_stream_read(&stream->istream) > 0);
 
446
 
 
447
                i_stream_seek(&stream->istream, old_offset);
 
448
                if (zstream->stream_size == (uoff_t)-1)
 
449
                        return -1;
 
450
        }
 
451
        stream->statbuf.st_size = zstream->stream_size;
 
452
        return 0;
 
453
}
 
454
 
 
455
static void i_stream_zlib_sync(struct istream_private *stream)
 
456
{
 
457
        struct zlib_istream *zstream = (struct zlib_istream *) stream;
 
458
        const struct stat *st;
 
459
 
 
460
        if (i_stream_stat(stream->parent, FALSE, &st) < 0) {
 
461
                if (memcmp(&zstream->last_parent_statbuf,
 
462
                           st, sizeof(*st)) == 0) {
 
463
                        /* a compressed file doesn't change unexpectedly,
 
464
                           don't clear our caches unnecessarily */
 
465
                        return;
 
466
                }
 
467
                zstream->last_parent_statbuf = *st;
 
468
        }
 
469
        i_stream_zlib_reset(zstream);
 
470
}
 
471
 
 
472
static struct istream *
 
473
i_stream_create_zlib(struct istream *input, bool gz, bool log_errors)
 
474
{
 
475
        struct zlib_istream *zstream;
 
476
 
 
477
        zstream = i_new(struct zlib_istream, 1);
 
478
        zstream->eof_offset = (uoff_t)-1;
 
479
        zstream->stream_size = (uoff_t)-1;
 
480
        zstream->gz = gz;
 
481
        zstream->log_errors = log_errors;
 
482
 
 
483
        i_stream_zlib_init(zstream);
 
484
 
 
485
        zstream->istream.iostream.close = i_stream_zlib_close;
 
486
        zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
 
487
        zstream->istream.read = i_stream_zlib_read;
 
488
        zstream->istream.seek = i_stream_zlib_seek;
 
489
        zstream->istream.stat = i_stream_zlib_stat;
 
490
        zstream->istream.sync = i_stream_zlib_sync;
 
491
 
 
492
        zstream->istream.istream.readable_fd = FALSE;
 
493
        zstream->istream.istream.blocking = input->blocking;
 
494
        zstream->istream.istream.seekable = input->seekable;
 
495
 
 
496
        return i_stream_create(&zstream->istream, input,
 
497
                               i_stream_get_fd(input));
 
498
}
 
499
 
 
500
struct istream *i_stream_create_gz(struct istream *input, bool log_errors)
 
501
{
 
502
        return i_stream_create_zlib(input, TRUE, log_errors);
 
503
}
 
504
 
 
505
struct istream *i_stream_create_deflate(struct istream *input, bool log_errors)
 
506
{
 
507
        return i_stream_create_zlib(input, FALSE, log_errors);
 
508
}
 
509
#endif