← Back to branch summary

~ubuntu-branches/ubuntu/wily/dovecot/wily

« back to all changes in this revision

Viewing changes to src/plugins/zlib/istream-zlib.c

  • Committer: Package Import Robot
  • Author(s): Jaldhar H. Vyas
  • Date: 2013-09-09 00:57:32 UTC
  • mfrom: (1.13.11)
  • mto: (4.8.5 experimental) (1.16.1)
  • mto: This revision was merged to the branch mainline in revision 97.
  • Revision ID: package-import@ubuntu.com-20130909005732-dn1eell8srqbhh0e
Tags: upstream-2.2.5
Import upstream version 2.2.5

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/plugins/zlib/istream-zlib.c
1
 
/* Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file */
2
 
 
3
 
#include "lib.h"
4
 
 
5
 
#ifdef HAVE_ZLIB
6
 
 
7
 
#include "crc32.h"
8
 
#include "istream-private.h"
9
 
#include "istream-zlib.h"
10
 
#include <zlib.h>
11
 
 
12
 
#define CHUNK_SIZE (1024*64)
13
 
 
14
 
#define GZ_HEADER_MIN_SIZE 10
15
 
#define GZ_TRAILER_SIZE 8
16
 
 
17
 
#define GZ_MAGIC1       0x1f
18
 
#define GZ_MAGIC2       0x8b
19
 
#define GZ_FLAG_FHCRC   0x02
20
 
#define GZ_FLAG_FEXTRA  0x04
21
 
#define GZ_FLAG_FNAME   0x08
22
 
#define GZ_FLAG_FCOMMENT 0x10
23
 
 
24
 
struct zlib_istream {
25
 
        struct istream_private istream;
26
 
 
27
 
        z_stream zs;
28
 
        uoff_t eof_offset, stream_size;
29
 
        size_t prev_size, high_pos;
30
 
        uint32_t crc32;
31
 
        struct stat last_parent_statbuf;
32
 
 
33
 
        unsigned int gz:1;
34
 
        unsigned int log_errors:1;
35
 
        unsigned int marked:1;
36
 
        unsigned int header_read:1;
37
 
        unsigned int trailer_read:1;
38
 
        unsigned int zs_closed:1;
39
 
};
40
 
 
41
 
static void i_stream_zlib_init(struct zlib_istream *zstream);
42
 
 
43
 
static void i_stream_zlib_close(struct iostream_private *stream)
44
 
{
45
 
        struct zlib_istream *zstream = (struct zlib_istream *)stream;
46
 
 
47
 
        if (!zstream->zs_closed) {
48
 
                (void)inflateEnd(&zstream->zs);
49
 
                zstream->zs_closed = TRUE;
50
 
        }
51
 
}
52
 
 
53
 
static void zlib_read_error(struct zlib_istream *zstream, const char *error)
54
 
{
55
 
        i_error("zlib.read(%s): %s at %"PRIuUOFF_T,
56
 
                i_stream_get_name(&zstream->istream.istream), error,
57
 
                zstream->istream.abs_start_offset +
58
 
                zstream->istream.istream.v_offset);
59
 
}
60
 
 
61
 
static int i_stream_zlib_read_header(struct istream_private *stream)
62
 
{
63
 
        struct zlib_istream *zstream = (struct zlib_istream *)stream;
64
 
        const unsigned char *data;
65
 
        size_t size;
66
 
        unsigned int pos, fextra_size;
67
 
        int ret;
68
 
 
69
 
        ret = i_stream_read_data(stream->parent, &data, &size,
70
 
                                 zstream->prev_size);
71
 
        if (size == zstream->prev_size) {
72
 
                if (ret == -1) {
73
 
                        if (zstream->log_errors)
74
 
                                zlib_read_error(zstream, "missing gz header");
75
 
                        stream->istream.stream_errno = EINVAL;
76
 
                }
77
 
                return ret;
78
 
        }
79
 
        zstream->prev_size = size;
80
 
 
81
 
        if (size < GZ_HEADER_MIN_SIZE)
82
 
                return 0;
83
 
        pos = GZ_HEADER_MIN_SIZE;
84
 
 
85
 
        if (data[0] != GZ_MAGIC1 || data[1] != GZ_MAGIC2) {
86
 
                /* missing gzip magic header */
87
 
                if (zstream->log_errors) {
88
 
                        zlib_read_error(zstream, "wrong magic in header "
89
 
                                        "(not gz file?)");
90
 
                }
91
 
                stream->istream.stream_errno = EINVAL;
92
 
                return -1;
93
 
        }
94
 
        if ((data[3] & GZ_FLAG_FEXTRA) != 0) {
95
 
                if (pos + 2 < size)
96
 
                        return 0;
97
 
 
98
 
                fextra_size = data[pos] + (data[pos+1] << 8);
99
 
                pos += 2;
100
 
                if (pos + fextra_size < size)
101
 
                        return 0;
102
 
                pos += fextra_size;
103
 
        }
104
 
        if ((data[3] & GZ_FLAG_FNAME) != 0) {
105
 
                do {
106
 
                        if (pos == size)
107
 
                                return 0;
108
 
                } while (data[pos++] != '\0');
109
 
        }
110
 
        if ((data[3] & GZ_FLAG_FCOMMENT) != 0) {
111
 
                do {
112
 
                        if (pos == size)
113
 
                                return 0;
114
 
                } while (data[pos++] != '\0');
115
 
        }
116
 
        if ((data[3] & GZ_FLAG_FHCRC) != 0) {
117
 
                if (pos + 2 < size)
118
 
                        return 0;
119
 
                pos += 2;
120
 
        }
121
 
        i_stream_skip(stream->parent, pos);
122
 
        zstream->prev_size = 0;
123
 
        return 1;
124
 
}
125
 
 
126
 
static uint32_t data_get_uint32(const unsigned char *data)
127
 
{
128
 
        return data[0] | (data[1] << 8) | (data[2] << 16) |
129
 
                ((uint32_t)data[3] << 24);
130
 
}
131
 
 
132
 
static int i_stream_zlib_read_trailer(struct zlib_istream *zstream)
133
 
{
134
 
        struct istream_private *stream = &zstream->istream;
135
 
        const unsigned char *data;
136
 
        size_t size;
137
 
        int ret;
138
 
 
139
 
        ret = i_stream_read_data(stream->parent, &data, &size,
140
 
                                 GZ_TRAILER_SIZE-1);
141
 
        if (size == zstream->prev_size) {
142
 
                if (ret == -1) {
143
 
                        if (zstream->log_errors)
144
 
                                zlib_read_error(zstream, "missing gz trailer");
145
 
                        stream->istream.stream_errno = EINVAL;
146
 
                }
147
 
                return ret;
148
 
        }
149
 
        zstream->prev_size = size;
150
 
 
151
 
        if (size < GZ_TRAILER_SIZE)
152
 
                return 0;
153
 
 
154
 
        if (data_get_uint32(data) != zstream->crc32) {
155
 
                if (zstream->log_errors) {
156
 
                        zlib_read_error(zstream,
157
 
                                        "gz trailer has wrong CRC value");
158
 
                }
159
 
                stream->istream.stream_errno = EINVAL;
160
 
                return -1;
161
 
        }
162
 
        i_stream_skip(stream->parent, GZ_TRAILER_SIZE);
163
 
        zstream->prev_size = 0;
164
 
        zstream->trailer_read = TRUE;
165
 
        return 1;
166
 
}
167
 
 
168
 
static ssize_t i_stream_zlib_read(struct istream_private *stream)
169
 
{
170
 
        struct zlib_istream *zstream = (struct zlib_istream *)stream;
171
 
        const unsigned char *data;
172
 
        uoff_t high_offset;
173
 
        size_t size, out_size;
174
 
        int ret;
175
 
 
176
 
        high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
177
 
        if (zstream->eof_offset == high_offset) {
178
 
                i_assert(zstream->high_pos == 0 ||
179
 
                         zstream->high_pos == stream->pos);
180
 
                if (!zstream->trailer_read) {
181
 
                        do {
182
 
                                ret = i_stream_zlib_read_trailer(zstream);
183
 
                        } while (ret == 0 && stream->istream.blocking);
184
 
                        if (ret <= 0)
185
 
                                return ret;
186
 
                }
187
 
                if (!zstream->gz || i_stream_is_eof(stream->parent)) {
188
 
                        stream->istream.eof = TRUE;
189
 
                        return -1;
190
 
                }
191
 
                /* gzip file with concatenated content */
192
 
                zstream->eof_offset = (uoff_t)-1;
193
 
                zstream->stream_size = (uoff_t)-1;
194
 
                zstream->header_read = FALSE;
195
 
                zstream->trailer_read = FALSE;
196
 
                zstream->crc32 = 0;
197
 
 
198
 
                (void)inflateEnd(&zstream->zs);
199
 
                i_stream_zlib_init(zstream);
200
 
        }
201
 
 
202
 
        if (!zstream->header_read) {
203
 
                do {
204
 
                        ret = i_stream_zlib_read_header(stream);
205
 
                } while (ret == 0 && stream->istream.blocking);
206
 
                if (ret <= 0)
207
 
                        return ret;
208
 
                zstream->header_read = TRUE;
209
 
        }
210
 
 
211
 
        if (stream->pos < zstream->high_pos) {
212
 
                /* we're here because we seeked back within the read buffer. */
213
 
                ret = zstream->high_pos - stream->pos;
214
 
                stream->pos = zstream->high_pos;
215
 
                zstream->high_pos = 0;
216
 
                if (zstream->trailer_read) {
217
 
                        high_offset = stream->istream.v_offset +
218
 
                                (stream->pos - stream->skip);
219
 
                        i_assert(zstream->eof_offset == high_offset);
220
 
                        stream->istream.eof = TRUE;
221
 
                }
222
 
                return ret;
223
 
        }
224
 
        zstream->high_pos = 0;
225
 
 
226
 
        if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
227
 
                /* try to keep at least CHUNK_SIZE available */
228
 
                if (!zstream->marked && stream->skip > 0) {
229
 
                        /* don't try to keep anything cached if we don't
230
 
                           have a seek mark. */
231
 
                        i_stream_compress(stream);
232
 
                }
233
 
                if (stream->max_buffer_size == 0 ||
234
 
                    stream->buffer_size < stream->max_buffer_size)
235
 
                        i_stream_grow_buffer(stream, CHUNK_SIZE);
236
 
 
237
 
                if (stream->pos == stream->buffer_size) {
238
 
                        if (stream->skip > 0) {
239
 
                                /* lose our buffer cache */
240
 
                                i_stream_compress(stream);
241
 
                        }
242
 
 
243
 
                        if (stream->pos == stream->buffer_size)
244
 
                                return -2; /* buffer full */
245
 
                }
246
 
        }
247
 
 
248
 
        if (i_stream_read_data(stream->parent, &data, &size, 0) < 0) {
249
 
                if (stream->parent->stream_errno != 0) {
250
 
                        stream->istream.stream_errno =
251
 
                                stream->parent->stream_errno;
252
 
                } else {
253
 
                        i_assert(stream->parent->eof);
254
 
                        if (zstream->log_errors)
255
 
                                zlib_read_error(zstream, "unexpected EOF");
256
 
                        stream->istream.stream_errno = EPIPE;
257
 
                }
258
 
                return -1;
259
 
        }
260
 
        if (size == 0) {
261
 
                /* no more input */
262
 
                i_assert(!stream->istream.blocking);
263
 
                return 0;
264
 
        }
265
 
 
266
 
        zstream->zs.next_in = (void *)data;
267
 
        zstream->zs.avail_in = size;
268
 
 
269
 
        out_size = stream->buffer_size - stream->pos;
270
 
        zstream->zs.next_out = stream->w_buffer + stream->pos;
271
 
        zstream->zs.avail_out = out_size;
272
 
        ret = inflate(&zstream->zs, Z_SYNC_FLUSH);
273
 
 
274
 
        out_size -= zstream->zs.avail_out;
275
 
        zstream->crc32 = crc32_data_more(zstream->crc32,
276
 
                                         stream->w_buffer + stream->pos,
277
 
                                         out_size);
278
 
        stream->pos += out_size;
279
 
 
280
 
        i_stream_skip(stream->parent, size - zstream->zs.avail_in);
281
 
 
282
 
        switch (ret) {
283
 
        case Z_OK:
284
 
                break;
285
 
        case Z_NEED_DICT:
286
 
                if (zstream->log_errors)
287
 
                        zlib_read_error(zstream, "can't read file without dict");
288
 
                stream->istream.stream_errno = EINVAL;
289
 
                return -1;
290
 
        case Z_DATA_ERROR:
291
 
                if (zstream->log_errors)
292
 
                        zlib_read_error(zstream, "corrupted data");
293
 
                stream->istream.stream_errno = EINVAL;
294
 
                return -1;
295
 
        case Z_MEM_ERROR:
296
 
                i_fatal_status(FATAL_OUTOFMEM, "zlib.read(%s): Out of memory",
297
 
                               i_stream_get_name(&stream->istream));
298
 
        case Z_STREAM_END:
299
 
                zstream->eof_offset = stream->istream.v_offset +
300
 
                        (stream->pos - stream->skip);
301
 
                zstream->stream_size = zstream->eof_offset;
302
 
                zstream->zs.avail_in = 0;
303
 
 
304
 
                if (!zstream->trailer_read) {
305
 
                        /* try to read and verify the trailer, we might not
306
 
                           be called again. */
307
 
                        if (i_stream_zlib_read_trailer(zstream) < 0)
308
 
                                return -1;
309
 
                }
310
 
                break;
311
 
        default:
312
 
                i_fatal("inflate() failed with %d", ret);
313
 
        }
314
 
        if (out_size == 0) {
315
 
                /* read more input */
316
 
                return i_stream_zlib_read(stream);
317
 
        }
318
 
        return out_size;
319
 
}
320
 
 
321
 
static void i_stream_zlib_init(struct zlib_istream *zstream)
322
 
{
323
 
        int ret;
324
 
 
325
 
        ret = inflateInit2(&zstream->zs, -15);
326
 
        switch (ret) {
327
 
        case Z_OK:
328
 
                break;
329
 
        case Z_MEM_ERROR:
330
 
                i_fatal_status(FATAL_OUTOFMEM, "zlib: Out of memory");
331
 
        case Z_VERSION_ERROR:
332
 
                i_fatal("Wrong zlib library version (broken compilation)");
333
 
        case Z_STREAM_ERROR:
334
 
                i_fatal("zlib: Invalid parameters");
335
 
        default:
336
 
                i_fatal("inflateInit() failed with %d", ret);
337
 
        }
338
 
        zstream->header_read = !zstream->gz;
339
 
        zstream->trailer_read = !zstream->gz;
340
 
}
341
 
 
342
 
static void i_stream_zlib_reset(struct zlib_istream *zstream)
343
 
{
344
 
        struct istream_private *stream = &zstream->istream;
345
 
 
346
 
        i_stream_seek(stream->parent, stream->parent_start_offset);
347
 
        zstream->eof_offset = (uoff_t)-1;
348
 
        zstream->crc32 = 0;
349
 
 
350
 
        zstream->zs.next_in = NULL;
351
 
        zstream->zs.avail_in = 0;
352
 
 
353
 
        stream->parent_expected_offset = stream->parent_start_offset;
354
 
        stream->skip = stream->pos = 0;
355
 
        stream->istream.v_offset = 0;
356
 
        zstream->high_pos = 0;
357
 
        zstream->prev_size = 0;
358
 
 
359
 
        (void)inflateEnd(&zstream->zs);
360
 
        i_stream_zlib_init(zstream);
361
 
}
362
 
 
363
 
static void
364
 
i_stream_zlib_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
365
 
{
366
 
        struct zlib_istream *zstream = (struct zlib_istream *) stream;
367
 
        uoff_t start_offset = stream->istream.v_offset - stream->skip;
368
 
 
369
 
        if (v_offset < start_offset) {
370
 
                /* have to seek backwards */
371
 
                i_stream_zlib_reset(zstream);
372
 
                start_offset = 0;
373
 
        } else if (zstream->high_pos != 0) {
374
 
                stream->pos = zstream->high_pos;
375
 
                zstream->high_pos = 0;
376
 
        }
377
 
 
378
 
        if (v_offset <= start_offset + stream->pos) {
379
 
                /* seeking backwards within what's already cached */
380
 
                stream->skip = v_offset - start_offset;
381
 
                stream->istream.v_offset = v_offset;
382
 
                zstream->high_pos = stream->pos;
383
 
                stream->pos = stream->skip;
384
 
        } else {
385
 
                /* read and cache forward */
386
 
                do {
387
 
                        size_t avail = stream->pos - stream->skip;
388
 
 
389
 
                        if (stream->istream.v_offset + avail >= v_offset) {
390
 
                                i_stream_skip(&stream->istream,
391
 
                                              v_offset -
392
 
                                              stream->istream.v_offset);
393
 
                                break;
394
 
                        }
395
 
 
396
 
                        i_stream_skip(&stream->istream, avail);
397
 
                } while (i_stream_read(&stream->istream) >= 0);
398
 
 
399
 
                if (stream->istream.v_offset != v_offset) {
400
 
                        /* some failure, we've broken it */
401
 
                        if (stream->istream.stream_errno != 0) {
402
 
                                i_error("zlib_istream.seek(%s) failed: %s",
403
 
                                        i_stream_get_name(&stream->istream),
404
 
                                        strerror(stream->istream.stream_errno));
405
 
                                i_stream_close(&stream->istream);
406
 
                        } else {
407
 
                                /* unexpected EOF. allow it since we may just
408
 
                                   want to check if there's anything.. */
409
 
                                i_assert(stream->istream.eof);
410
 
                        }
411
 
                }
412
 
        }
413
 
 
414
 
        if (mark)
415
 
                zstream->marked = TRUE;
416
 
}
417
 
 
418
 
static const struct stat *
419
 
i_stream_zlib_stat(struct istream_private *stream, bool exact)
420
 
{
421
 
        struct zlib_istream *zstream = (struct zlib_istream *) stream;
422
 
        const struct stat *st;
423
 
        size_t size;
424
 
 
425
 
        st = i_stream_stat(stream->parent, exact);
426
 
        if (st == NULL)
427
 
                return NULL;
428
 
 
429
 
        /* when exact=FALSE always return the parent stat's size, even if we
430
 
           know the exact value. this is necessary because otherwise e.g. mbox
431
 
           code can see two different values and think that a compressed mbox
432
 
           file keeps changing. */
433
 
        if (!exact)
434
 
                return st;
435
 
 
436
 
        stream->statbuf = *st;
437
 
        if (zstream->stream_size == (uoff_t)-1) {
438
 
                uoff_t old_offset = stream->istream.v_offset;
439
 
 
440
 
                do {
441
 
                        (void)i_stream_get_data(&stream->istream, &size);
442
 
                        i_stream_skip(&stream->istream, size);
443
 
                } while (i_stream_read(&stream->istream) > 0);
444
 
 
445
 
                i_stream_seek(&stream->istream, old_offset);
446
 
                if (zstream->stream_size == (uoff_t)-1)
447
 
                        return NULL;
448
 
        }
449
 
        stream->statbuf.st_size = zstream->stream_size;
450
 
        return &stream->statbuf;
451
 
}
452
 
 
453
 
static void i_stream_zlib_sync(struct istream_private *stream)
454
 
{
455
 
        struct zlib_istream *zstream = (struct zlib_istream *) stream;
456
 
        const struct stat *st;
457
 
 
458
 
        st = i_stream_stat(stream->parent, FALSE);
459
 
        if (st != NULL) {
460
 
                if (memcmp(&zstream->last_parent_statbuf,
461
 
                           st, sizeof(*st)) == 0) {
462
 
                        /* a compressed file doesn't change unexpectedly,
463
 
                           don't clear our caches unnecessarily */
464
 
                        return;
465
 
                }
466
 
                zstream->last_parent_statbuf = *st;
467
 
        }
468
 
        i_stream_zlib_reset(zstream);
469
 
}
470
 
 
471
 
static struct istream *
472
 
i_stream_create_zlib(struct istream *input, bool gz, bool log_errors)
473
 
{
474
 
        struct zlib_istream *zstream;
475
 
 
476
 
        zstream = i_new(struct zlib_istream, 1);
477
 
        zstream->eof_offset = (uoff_t)-1;
478
 
        zstream->stream_size = (uoff_t)-1;
479
 
        zstream->gz = gz;
480
 
        zstream->log_errors = log_errors;
481
 
 
482
 
        i_stream_zlib_init(zstream);
483
 
 
484
 
        zstream->istream.iostream.close = i_stream_zlib_close;
485
 
        zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
486
 
        zstream->istream.read = i_stream_zlib_read;
487
 
        zstream->istream.seek = i_stream_zlib_seek;
488
 
        zstream->istream.stat = i_stream_zlib_stat;
489
 
        zstream->istream.sync = i_stream_zlib_sync;
490
 
 
491
 
        zstream->istream.istream.readable_fd = FALSE;
492
 
        zstream->istream.istream.blocking = input->blocking;
493
 
        zstream->istream.istream.seekable = input->seekable;
494
 
 
495
 
        return i_stream_create(&zstream->istream, input,
496
 
                               i_stream_get_fd(input));
497
 
}
498
 
 
499
 
struct istream *i_stream_create_gz(struct istream *input, bool log_errors)
500
 
{
501
 
        return i_stream_create_zlib(input, TRUE, log_errors);
502
 
}
503
 
 
504
 
struct istream *i_stream_create_deflate(struct istream *input, bool log_errors)
505
 
{
506
 
        return i_stream_create_zlib(input, FALSE, log_errors);
507
 
}
508
 
#endif