← Back to branch summary

~ubuntu-branches/ubuntu/natty/tiff/natty-security

« back to all changes in this revision

Viewing changes to .pc/CVE-2012-2088.patch/libtiff/tif_strip.c

  • Committer: Package Import Robot
  • Author(s): Marc Deslauriers
  • Date: 2012-07-04 10:59:25 UTC
  • Revision ID: package-import@ubuntu.com-20120704105925-codjws8elgvml0lf
Tags: 3.9.4-5ubuntu6.2
https://launchpad.net/bugs/1016324
https://launchpad.net/bugs/1016324
* SECURITY UPDATE: possible arbitrary code execution via buffer overflow
  due to type-conversion flaw (LP: #1016324)
  - debian/patches/CVE-2012-2088.patch: check for overflows in
    libtiff/tif_strip.c and libtiff/tif_tile.c.
  - CVE-2012-2088
* SECURITY UPDATE: possible arbitrary code execution via integer
  overflows in tiff2pdf (LP: #1016324)
  - debian/patches/CVE-2012-2113.patch: check for overflows in
    tools/tiff2pdf.c.
  - CVE-2012-2113

Show diffs side-by-side

added added

removed removed

Lines of Context:
.pc/CVE-2012-2088.patch/libtiff/tif_strip.c
 
1
/* $Id: tif_strip.c,v 1.19.2.1 2010-06-08 18:50:43 bfriesen Exp $ */
 
2
 
 
3
/*
 
4
 * Copyright (c) 1991-1997 Sam Leffler
 
5
 * Copyright (c) 1991-1997 Silicon Graphics, Inc.
 
6
 *
 
7
 * Permission to use, copy, modify, distribute, and sell this software and 
 
8
 * its documentation for any purpose is hereby granted without fee, provided
 
9
 * that (i) the above copyright notices and this permission notice appear in
 
10
 * all copies of the software and related documentation, and (ii) the names of
 
11
 * Sam Leffler and Silicon Graphics may not be used in any advertising or
 
12
 * publicity relating to the software without the specific, prior written
 
13
 * permission of Sam Leffler and Silicon Graphics.
 
14
 * 
 
15
 * THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND, 
 
16
 * EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY 
 
17
 * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.  
 
18
 * 
 
19
 * IN NO EVENT SHALL SAM LEFFLER OR SILICON GRAPHICS BE LIABLE FOR
 
20
 * ANY SPECIAL, INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND,
 
21
 * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
 
22
 * WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF 
 
23
 * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE 
 
24
 * OF THIS SOFTWARE.
 
25
 */
 
26
 
 
27
/*
 
28
 * TIFF Library.
 
29
 *
 
30
 * Strip-organized Image Support Routines.
 
31
 */
 
32
#include "tiffiop.h"
 
33
 
 
34
static uint32
 
35
summarize(TIFF* tif, size_t summand1, size_t summand2, const char* where)
 
36
{
 
37
        /*
 
38
         * XXX: We are using casting to uint32 here, bacause sizeof(size_t)
 
39
         * may be larger than sizeof(uint32) on 64-bit architectures.
 
40
         */
 
41
        uint32  bytes = summand1 + summand2;
 
42
 
 
43
        if (bytes - summand1 != summand2) {
 
44
                TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", where);
 
45
                bytes = 0;
 
46
        }
 
47
 
 
48
        return (bytes);
 
49
}
 
50
 
 
51
static uint32
 
52
multiply(TIFF* tif, size_t nmemb, size_t elem_size, const char* where)
 
53
{
 
54
        uint32  bytes = nmemb * elem_size;
 
55
 
 
56
        if (elem_size && bytes / elem_size != nmemb) {
 
57
                TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", where);
 
58
                bytes = 0;
 
59
        }
 
60
 
 
61
        return (bytes);
 
62
}
 
63
 
 
64
/*
 
65
 * Compute which strip a (row,sample) value is in.
 
66
 */
 
67
tstrip_t
 
68
TIFFComputeStrip(TIFF* tif, uint32 row, tsample_t sample)
 
69
{
 
70
        TIFFDirectory *td = &tif->tif_dir;
 
71
        tstrip_t strip;
 
72
 
 
73
        strip = row / td->td_rowsperstrip;
 
74
        if (td->td_planarconfig == PLANARCONFIG_SEPARATE) {
 
75
                if (sample >= td->td_samplesperpixel) {
 
76
                        TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
 
77
                            "%lu: Sample out of range, max %lu",
 
78
                            (unsigned long) sample, (unsigned long) td->td_samplesperpixel);
 
79
                        return ((tstrip_t) 0);
 
80
                }
 
81
                strip += sample*td->td_stripsperimage;
 
82
        }
 
83
        return (strip);
 
84
}
 
85
 
 
86
/*
 
87
 * Compute how many strips are in an image.
 
88
 */
 
89
tstrip_t
 
90
TIFFNumberOfStrips(TIFF* tif)
 
91
{
 
92
        TIFFDirectory *td = &tif->tif_dir;
 
93
        tstrip_t nstrips;
 
94
 
 
95
        nstrips = (td->td_rowsperstrip == (uint32) -1 ? 1 :
 
96
             TIFFhowmany(td->td_imagelength, td->td_rowsperstrip));
 
97
        if (td->td_planarconfig == PLANARCONFIG_SEPARATE)
 
98
                nstrips = multiply(tif, nstrips, td->td_samplesperpixel,
 
99
                                   "TIFFNumberOfStrips");
 
100
        return (nstrips);
 
101
}
 
102
 
 
103
/*
 
104
 * Compute the # bytes in a variable height, row-aligned strip.
 
105
 */
 
106
tsize_t
 
107
TIFFVStripSize(TIFF* tif, uint32 nrows)
 
108
{
 
109
        TIFFDirectory *td = &tif->tif_dir;
 
110
 
 
111
        if (nrows == (uint32) -1)
 
112
                nrows = td->td_imagelength;
 
113
        if (td->td_planarconfig == PLANARCONFIG_CONTIG &&
 
114
            td->td_photometric == PHOTOMETRIC_YCBCR &&
 
115
            !isUpSampled(tif)) {
 
116
                /*
 
117
                 * Packed YCbCr data contain one Cb+Cr for every
 
118
                 * HorizontalSampling*VerticalSampling Y values.
 
119
                 * Must also roundup width and height when calculating
 
120
                 * since images that are not a multiple of the
 
121
                 * horizontal/vertical subsampling area include
 
122
                 * YCbCr data for the extended image.
 
123
                 */
 
124
                uint16 ycbcrsubsampling[2];
 
125
                tsize_t w, scanline, samplingarea;
 
126
 
 
127
                TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
 
128
                                      ycbcrsubsampling + 0,
 
129
                                      ycbcrsubsampling + 1);
 
130
 
 
131
                samplingarea = ycbcrsubsampling[0]*ycbcrsubsampling[1];
 
132
                if (samplingarea == 0) {
 
133
                        TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
 
134
                                     "Invalid YCbCr subsampling");
 
135
                        return 0;
 
136
                }
 
137
 
 
138
                w = TIFFroundup(td->td_imagewidth, ycbcrsubsampling[0]);
 
139
                scanline = TIFFhowmany8(multiply(tif, w, td->td_bitspersample,
 
140
                                                 "TIFFVStripSize"));
 
141
                nrows = TIFFroundup(nrows, ycbcrsubsampling[1]);
 
142
                /* NB: don't need TIFFhowmany here 'cuz everything is rounded */
 
143
                scanline = multiply(tif, nrows, scanline, "TIFFVStripSize");
 
144
                return ((tsize_t)
 
145
                    summarize(tif, scanline,
 
146
                              multiply(tif, 2, scanline / samplingarea,
 
147
                                       "TIFFVStripSize"), "TIFFVStripSize"));
 
148
        } else
 
149
                return ((tsize_t) multiply(tif, nrows, TIFFScanlineSize(tif),
 
150
                                           "TIFFVStripSize"));
 
151
}
 
152
 
 
153
 
 
154
/*
 
155
 * Compute the # bytes in a raw strip.
 
156
 */
 
157
tsize_t
 
158
TIFFRawStripSize(TIFF* tif, tstrip_t strip)
 
159
{
 
160
        TIFFDirectory* td = &tif->tif_dir;
 
161
        tsize_t bytecount = td->td_stripbytecount[strip];
 
162
 
 
163
        if (bytecount <= 0) {
 
164
                TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
 
165
                          "%lu: Invalid strip byte count, strip %lu",
 
166
                          (unsigned long) bytecount, (unsigned long) strip);
 
167
                bytecount = (tsize_t) -1;
 
168
        }
 
169
 
 
170
        return bytecount;
 
171
}
 
172
 
 
173
/*
 
174
 * Compute the # bytes in a (row-aligned) strip.
 
175
 *
 
176
 * Note that if RowsPerStrip is larger than the
 
177
 * recorded ImageLength, then the strip size is
 
178
 * truncated to reflect the actual space required
 
179
 * to hold the strip.
 
180
 */
 
181
tsize_t
 
182
TIFFStripSize(TIFF* tif)
 
183
{
 
184
        TIFFDirectory* td = &tif->tif_dir;
 
185
        uint32 rps = td->td_rowsperstrip;
 
186
        if (rps > td->td_imagelength)
 
187
                rps = td->td_imagelength;
 
188
        return (TIFFVStripSize(tif, rps));
 
189
}
 
190
 
 
191
/*
 
192
 * Compute a default strip size based on the image
 
193
 * characteristics and a requested value.  If the
 
194
 * request is <1 then we choose a strip size according
 
195
 * to certain heuristics.
 
196
 */
 
197
uint32
 
198
TIFFDefaultStripSize(TIFF* tif, uint32 request)
 
199
{
 
200
        return (*tif->tif_defstripsize)(tif, request);
 
201
}
 
202
 
 
203
uint32
 
204
_TIFFDefaultStripSize(TIFF* tif, uint32 s)
 
205
{
 
206
        if ((int32) s < 1) {
 
207
                /*
 
208
                 * If RowsPerStrip is unspecified, try to break the
 
209
                 * image up into strips that are approximately
 
210
                 * STRIP_SIZE_DEFAULT bytes long.
 
211
                 */
 
212
                tsize_t scanline = TIFFScanlineSize(tif);
 
213
                s = (uint32)STRIP_SIZE_DEFAULT / (scanline == 0 ? 1 : scanline);
 
214
                if (s == 0)             /* very wide images */
 
215
                        s = 1;
 
216
        }
 
217
        return (s);
 
218
}
 
219
 
 
220
/*
 
221
 * Return the number of bytes to read/write in a call to
 
222
 * one of the scanline-oriented i/o routines.  Note that
 
223
 * this number may be 1/samples-per-pixel if data is
 
224
 * stored as separate planes.
 
225
 */
 
226
tsize_t
 
227
TIFFScanlineSize(TIFF* tif)
 
228
{
 
229
        TIFFDirectory *td = &tif->tif_dir;
 
230
        tsize_t scanline;
 
231
 
 
232
        if (td->td_planarconfig == PLANARCONFIG_CONTIG) {
 
233
                if (td->td_photometric == PHOTOMETRIC_YCBCR
 
234
                    && !isUpSampled(tif)) {
 
235
                        uint16 ycbcrsubsampling[2];
 
236
 
 
237
                        TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
 
238
                                              ycbcrsubsampling + 0,
 
239
                                              ycbcrsubsampling + 1);
 
240
 
 
241
                        if (ycbcrsubsampling[0]*ycbcrsubsampling[1] == 0) {
 
242
                                TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
 
243
                                             "Invalid YCbCr subsampling");
 
244
                                return 0;
 
245
                        }
 
246
 
 
247
                        /* number of sample clumps per line */
 
248
                        scanline = TIFFhowmany(td->td_imagewidth,
 
249
                                               ycbcrsubsampling[0]);
 
250
                        /* number of samples per line */
 
251
                        scanline = multiply(tif, scanline,
 
252
                                            ycbcrsubsampling[0]*ycbcrsubsampling[1] + 2,
 
253
                                            "TIFFScanlineSize");
 
254
                } else {
 
255
                        scanline = multiply(tif, td->td_imagewidth,
 
256
                                            td->td_samplesperpixel,
 
257
                                            "TIFFScanlineSize");
 
258
                }
 
259
        } else
 
260
                scanline = td->td_imagewidth;
 
261
        return ((tsize_t) TIFFhowmany8(multiply(tif, scanline,
 
262
                                                td->td_bitspersample,
 
263
                                                "TIFFScanlineSize")));
 
264
}
 
265
 
 
266
/*
 
267
 * Some stuff depends on this older version of TIFFScanlineSize
 
268
 * TODO: resolve this
 
269
 */
 
270
tsize_t
 
271
TIFFOldScanlineSize(TIFF* tif)
 
272
{
 
273
        TIFFDirectory *td = &tif->tif_dir;
 
274
        tsize_t scanline;
 
275
 
 
276
        scanline = multiply (tif, td->td_bitspersample, td->td_imagewidth,
 
277
                             "TIFFScanlineSize");
 
278
        if (td->td_planarconfig == PLANARCONFIG_CONTIG)
 
279
                scanline = multiply (tif, scanline, td->td_samplesperpixel,
 
280
                                     "TIFFScanlineSize");
 
281
        return ((tsize_t) TIFFhowmany8(scanline));
 
282
}
 
283
 
 
284
/*
 
285
 * Return the number of bytes to read/write in a call to
 
286
 * one of the scanline-oriented i/o routines.  Note that
 
287
 * this number may be 1/samples-per-pixel if data is
 
288
 * stored as separate planes.
 
289
 * The ScanlineSize in case of YCbCrSubsampling is defined as the
 
290
 * strip size divided by the strip height, i.e. the size of a pack of vertical
 
291
 * subsampling lines divided by vertical subsampling. It should thus make
 
292
 * sense when multiplied by a multiple of vertical subsampling.
 
293
 * Some stuff depends on this newer version of TIFFScanlineSize
 
294
 * TODO: resolve this
 
295
 */
 
296
tsize_t
 
297
TIFFNewScanlineSize(TIFF* tif)
 
298
{
 
299
        TIFFDirectory *td = &tif->tif_dir;
 
300
        tsize_t scanline;
 
301
 
 
302
        if (td->td_planarconfig == PLANARCONFIG_CONTIG) {
 
303
                if (td->td_photometric == PHOTOMETRIC_YCBCR
 
304
                    && !isUpSampled(tif)) {
 
305
                        uint16 ycbcrsubsampling[2];
 
306
 
 
307
                        TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
 
308
                                              ycbcrsubsampling + 0,
 
309
                                              ycbcrsubsampling + 1);
 
310
 
 
311
                        if (ycbcrsubsampling[0]*ycbcrsubsampling[1] == 0) {
 
312
                                TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
 
313
                                             "Invalid YCbCr subsampling");
 
314
                                return 0;
 
315
                        }
 
316
 
 
317
                        return((tsize_t) ((((td->td_imagewidth+ycbcrsubsampling[0]-1)
 
318
                                            /ycbcrsubsampling[0])
 
319
                                           *(ycbcrsubsampling[0]*ycbcrsubsampling[1]+2)
 
320
                                           *td->td_bitspersample+7)
 
321
                                          /8)/ycbcrsubsampling[1]);
 
322
 
 
323
                } else {
 
324
                        scanline = multiply(tif, td->td_imagewidth,
 
325
                                            td->td_samplesperpixel,
 
326
                                            "TIFFScanlineSize");
 
327
                }
 
328
        } else
 
329
                scanline = td->td_imagewidth;
 
330
        return ((tsize_t) TIFFhowmany8(multiply(tif, scanline,
 
331
                                                td->td_bitspersample,
 
332
                                                "TIFFScanlineSize")));
 
333
}
 
334
 
 
335
/*
 
336
 * Return the number of bytes required to store a complete
 
337
 * decoded and packed raster scanline (as opposed to the
 
338
 * I/O size returned by TIFFScanlineSize which may be less
 
339
 * if data is store as separate planes).
 
340
 */
 
341
tsize_t
 
342
TIFFRasterScanlineSize(TIFF* tif)
 
343
{
 
344
        TIFFDirectory *td = &tif->tif_dir;
 
345
        tsize_t scanline;
 
346
        
 
347
        scanline = multiply (tif, td->td_bitspersample, td->td_imagewidth,
 
348
                             "TIFFRasterScanlineSize");
 
349
        if (td->td_planarconfig == PLANARCONFIG_CONTIG) {
 
350
                scanline = multiply (tif, scanline, td->td_samplesperpixel,
 
351
                                     "TIFFRasterScanlineSize");
 
352
                return ((tsize_t) TIFFhowmany8(scanline));
 
353
        } else
 
354
                return ((tsize_t) multiply (tif, TIFFhowmany8(scanline),
 
355
                                            td->td_samplesperpixel,
 
356
                                            "TIFFRasterScanlineSize"));
 
357
}
 
358
 
 
359
/* vim: set ts=8 sts=8 sw=8 noet: */
 
360
/*
 
361
 * Local Variables:
 
362
 * mode: c
 
363
 * c-basic-offset: 8
 
364
 * fill-column: 78
 
365
 * End:
 
366
 */