1
by Chuck Short
Initial version |
1 |
/* gzjoin -- command to join gzip files into one gzip file
|
2 |
||
3 |
Copyright (C) 2004 Mark Adler, all rights reserved
|
|
4 |
version 1.0, 11 Dec 2004
|
|
5 |
||
6 |
This software is provided 'as-is', without any express or implied
|
|
7 |
warranty. In no event will the author be held liable for any damages
|
|
8 |
arising from the use of this software.
|
|
9 |
||
10 |
Permission is granted to anyone to use this software for any purpose,
|
|
11 |
including commercial applications, and to alter it and redistribute it
|
|
12 |
freely, subject to the following restrictions:
|
|
13 |
||
14 |
1. The origin of this software must not be misrepresented; you must not
|
|
15 |
claim that you wrote the original software. If you use this software
|
|
16 |
in a product, an acknowledgment in the product documentation would be
|
|
17 |
appreciated but is not required.
|
|
18 |
2. Altered source versions must be plainly marked as such, and must not be
|
|
19 |
misrepresented as being the original software.
|
|
20 |
3. This notice may not be removed or altered from any source distribution.
|
|
21 |
||
22 |
Mark Adler madler@alumni.caltech.edu
|
|
23 |
*/
|
|
24 |
||
25 |
/*
|
|
26 |
* Change history:
|
|
27 |
*
|
|
28 |
* 1.0 11 Dec 2004 - First version
|
|
29 |
* 1.1 12 Jun 2005 - Changed ssize_t to long for portability
|
|
30 |
*/
|
|
31 |
||
32 |
/*
|
|
33 |
gzjoin takes one or more gzip files on the command line and writes out a
|
|
34 |
single gzip file that will uncompress to the concatenation of the
|
|
35 |
uncompressed data from the individual gzip files. gzjoin does this without
|
|
36 |
having to recompress any of the data and without having to calculate a new
|
|
37 |
crc32 for the concatenated uncompressed data. gzjoin does however have to
|
|
38 |
decompress all of the input data in order to find the bits in the compressed
|
|
39 |
data that need to be modified to concatenate the streams.
|
|
40 |
||
41 |
gzjoin does not do an integrity check on the input gzip files other than
|
|
42 |
checking the gzip header and decompressing the compressed data. They are
|
|
43 |
otherwise assumed to be complete and correct.
|
|
44 |
||
45 |
Each joint between gzip files removes at least 18 bytes of previous trailer
|
|
46 |
and subsequent header, and inserts an average of about three bytes to the
|
|
47 |
compressed data in order to connect the streams. The output gzip file
|
|
48 |
has a minimal ten-byte gzip header with no file name or modification time.
|
|
49 |
||
50 |
This program was written to illustrate the use of the Z_BLOCK option of
|
|
51 |
inflate() and the crc32_combine() function. gzjoin will not compile with
|
|
52 |
versions of zlib earlier than 1.2.3.
|
|
53 |
*/
|
|
54 |
||
55 |
#include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */ |
|
56 |
#include <stdlib.h> /* exit(), malloc(), free() */ |
|
57 |
#include <fcntl.h> /* open() */ |
|
58 |
#include <unistd.h> /* close(), read(), lseek() */ |
|
59 |
#include "zlib.h" |
|
60 |
/* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */
|
|
61 |
||
62 |
#define local static
|
|
63 |
||
64 |
/* exit with an error (return a value to allow use in an expression) */
|
|
65 |
local int bail(char *why1, char *why2) |
|
66 |
{
|
|
67 |
fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2); |
|
68 |
exit(1); |
|
69 |
return 0; |
|
70 |
}
|
|
71 |
||
72 |
/* -- simple buffered file input with access to the buffer -- */
|
|
73 |
||
74 |
#define CHUNK 32768 /* must be a power of two and fit in unsigned */ |
|
75 |
||
76 |
/* bin buffered input file type */
|
|
77 |
typedef struct { |
|
78 |
char *name; /* name of file for error messages */ |
|
79 |
int fd; /* file descriptor */ |
|
80 |
unsigned left; /* bytes remaining at next */ |
|
81 |
unsigned char *next; /* next byte to read */ |
|
82 |
unsigned char *buf; /* allocated buffer of length CHUNK */ |
|
83 |
} bin; |
|
84 |
||
85 |
/* close a buffered file and free allocated memory */
|
|
86 |
local void bclose(bin *in) |
|
87 |
{
|
|
88 |
if (in != NULL) { |
|
89 |
if (in->fd != -1) |
|
90 |
close(in->fd); |
|
91 |
if (in->buf != NULL) |
|
92 |
free(in->buf); |
|
93 |
free(in); |
|
94 |
}
|
|
95 |
}
|
|
96 |
||
97 |
/* open a buffered file for input, return a pointer to type bin, or NULL on
|
|
98 |
failure */
|
|
99 |
local bin *bopen(char *name) |
|
100 |
{
|
|
101 |
bin *in; |
|
102 |
||
103 |
in = malloc(sizeof(bin)); |
|
104 |
if (in == NULL) |
|
105 |
return NULL; |
|
106 |
in->buf = malloc(CHUNK); |
|
107 |
in->fd = open(name, O_RDONLY, 0); |
|
108 |
if (in->buf == NULL || in->fd == -1) { |
|
109 |
bclose(in); |
|
110 |
return NULL; |
|
111 |
}
|
|
112 |
in->left = 0; |
|
113 |
in->next = in->buf; |
|
114 |
in->name = name; |
|
115 |
return in; |
|
116 |
}
|
|
117 |
||
118 |
/* load buffer from file, return -1 on read error, 0 or 1 on success, with
|
|
119 |
1 indicating that end-of-file was reached */
|
|
120 |
local int bload(bin *in) |
|
121 |
{
|
|
122 |
long len; |
|
123 |
||
124 |
if (in == NULL) |
|
125 |
return -1; |
|
126 |
if (in->left != 0) |
|
127 |
return 0; |
|
128 |
in->next = in->buf; |
|
129 |
do { |
|
130 |
len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left); |
|
131 |
if (len < 0) |
|
132 |
return -1; |
|
133 |
in->left += (unsigned)len; |
|
134 |
} while (len != 0 && in->left < CHUNK); |
|
135 |
return len == 0 ? 1 : 0; |
|
136 |
}
|
|
137 |
||
138 |
/* get a byte from the file, bail if end of file */
|
|
139 |
#define bget(in) (in->left ? 0 : bload(in), \
|
|
140 |
in->left ? (in->left--, *(in->next)++) : \
|
|
141 |
bail("unexpected end of file on ", in->name))
|
|
142 |
||
143 |
/* get a four-byte little-endian unsigned integer from file */
|
|
144 |
local unsigned long bget4(bin *in) |
|
145 |
{
|
|
146 |
unsigned long val; |
|
147 |
||
148 |
val = bget(in); |
|
149 |
val += (unsigned long)(bget(in)) << 8; |
|
150 |
val += (unsigned long)(bget(in)) << 16; |
|
151 |
val += (unsigned long)(bget(in)) << 24; |
|
152 |
return val; |
|
153 |
}
|
|
154 |
||
155 |
/* skip bytes in file */
|
|
156 |
local void bskip(bin *in, unsigned skip) |
|
157 |
{
|
|
158 |
/* check pointer */
|
|
159 |
if (in == NULL) |
|
160 |
return; |
|
161 |
||
162 |
/* easy case -- skip bytes in buffer */
|
|
163 |
if (skip <= in->left) { |
|
164 |
in->left -= skip; |
|
165 |
in->next += skip; |
|
166 |
return; |
|
167 |
}
|
|
168 |
||
169 |
/* skip what's in buffer, discard buffer contents */
|
|
170 |
skip -= in->left; |
|
171 |
in->left = 0; |
|
172 |
||
173 |
/* seek past multiples of CHUNK bytes */
|
|
174 |
if (skip > CHUNK) { |
|
175 |
unsigned left; |
|
176 |
||
177 |
left = skip & (CHUNK - 1); |
|
178 |
if (left == 0) { |
|
179 |
/* exact number of chunks: seek all the way minus one byte to check
|
|
180 |
for end-of-file with a read */
|
|
181 |
lseek(in->fd, skip - 1, SEEK_CUR); |
|
182 |
if (read(in->fd, in->buf, 1) != 1) |
|
183 |
bail("unexpected end of file on ", in->name); |
|
184 |
return; |
|
185 |
}
|
|
186 |
||
187 |
/* skip the integral chunks, update skip with remainder */
|
|
188 |
lseek(in->fd, skip - left, SEEK_CUR); |
|
189 |
skip = left; |
|
190 |
}
|
|
191 |
||
192 |
/* read more input and skip remainder */
|
|
193 |
bload(in); |
|
194 |
if (skip > in->left) |
|
195 |
bail("unexpected end of file on ", in->name); |
|
196 |
in->left -= skip; |
|
197 |
in->next += skip; |
|
198 |
}
|
|
199 |
||
200 |
/* -- end of buffered input functions -- */
|
|
201 |
||
202 |
/* skip the gzip header from file in */
|
|
203 |
local void gzhead(bin *in) |
|
204 |
{
|
|
205 |
int flags; |
|
206 |
||
207 |
/* verify gzip magic header and compression method */
|
|
208 |
if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8) |
|
209 |
bail(in->name, " is not a valid gzip file"); |
|
210 |
||
211 |
/* get and verify flags */
|
|
212 |
flags = bget(in); |
|
213 |
if ((flags & 0xe0) != 0) |
|
214 |
bail("unknown reserved bits set in ", in->name); |
|
215 |
||
216 |
/* skip modification time, extra flags, and os */
|
|
217 |
bskip(in, 6); |
|
218 |
||
219 |
/* skip extra field if present */
|
|
220 |
if (flags & 4) { |
|
221 |
unsigned len; |
|
222 |
||
223 |
len = bget(in); |
|
224 |
len += (unsigned)(bget(in)) << 8; |
|
225 |
bskip(in, len); |
|
226 |
}
|
|
227 |
||
228 |
/* skip file name if present */
|
|
229 |
if (flags & 8) |
|
230 |
while (bget(in) != 0) |
|
231 |
;
|
|
232 |
||
233 |
/* skip comment if present */
|
|
234 |
if (flags & 16) |
|
235 |
while (bget(in) != 0) |
|
236 |
;
|
|
237 |
||
238 |
/* skip header crc if present */
|
|
239 |
if (flags & 2) |
|
240 |
bskip(in, 2); |
|
241 |
}
|
|
242 |
||
243 |
/* write a four-byte little-endian unsigned integer to out */
|
|
244 |
local void put4(unsigned long val, FILE *out) |
|
245 |
{
|
|
246 |
putc(val & 0xff, out); |
|
247 |
putc((val >> 8) & 0xff, out); |
|
248 |
putc((val >> 16) & 0xff, out); |
|
249 |
putc((val >> 24) & 0xff, out); |
|
250 |
}
|
|
251 |
||
252 |
/* Load up zlib stream from buffered input, bail if end of file */
|
|
253 |
local void zpull(z_streamp strm, bin *in) |
|
254 |
{
|
|
255 |
if (in->left == 0) |
|
256 |
bload(in); |
|
257 |
if (in->left == 0) |
|
258 |
bail("unexpected end of file on ", in->name); |
|
259 |
strm->avail_in = in->left; |
|
260 |
strm->next_in = in->next; |
|
261 |
}
|
|
262 |
||
263 |
/* Write header for gzip file to out and initialize trailer. */
|
|
264 |
local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out) |
|
265 |
{
|
|
266 |
fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out); |
|
267 |
*crc = crc32(0L, Z_NULL, 0); |
|
268 |
*tot = 0; |
|
269 |
}
|
|
270 |
||
271 |
/* Copy the compressed data from name, zeroing the last block bit of the last
|
|
272 |
block if clr is true, and adding empty blocks as needed to get to a byte
|
|
273 |
boundary. If clr is false, then the last block becomes the last block of
|
|
274 |
the output, and the gzip trailer is written. crc and tot maintains the
|
|
275 |
crc and length (modulo 2^32) of the output for the trailer. The resulting
|
|
276 |
gzip file is written to out. gzinit() must be called before the first call
|
|
277 |
of gzcopy() to write the gzip header and to initialize crc and tot. */
|
|
278 |
local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot, |
|
279 |
FILE *out) |
|
280 |
{
|
|
281 |
int ret; /* return value from zlib functions */ |
|
282 |
int pos; /* where the "last block" bit is in byte */ |
|
283 |
int last; /* true if processing the last block */ |
|
284 |
bin *in; /* buffered input file */ |
|
285 |
unsigned char *start; /* start of compressed data in buffer */ |
|
286 |
unsigned char *junk; /* buffer for uncompressed data -- discarded */ |
|
287 |
z_off_t len; /* length of uncompressed data (support > 4 GB) */ |
|
288 |
z_stream strm; /* zlib inflate stream */ |
|
289 |
||
290 |
/* open gzip file and skip header */
|
|
291 |
in = bopen(name); |
|
292 |
if (in == NULL) |
|
293 |
bail("could not open ", name); |
|
294 |
gzhead(in); |
|
295 |
||
296 |
/* allocate buffer for uncompressed data and initialize raw inflate
|
|
297 |
stream */
|
|
298 |
junk = malloc(CHUNK); |
|
299 |
strm.zalloc = Z_NULL; |
|
300 |
strm.zfree = Z_NULL; |
|
301 |
strm.opaque = Z_NULL; |
|
302 |
strm.avail_in = 0; |
|
303 |
strm.next_in = Z_NULL; |
|
304 |
ret = inflateInit2(&strm, -15); |
|
305 |
if (junk == NULL || ret != Z_OK) |
|
306 |
bail("out of memory", ""); |
|
307 |
||
308 |
/* inflate and copy compressed data, clear last-block bit if requested */
|
|
309 |
len = 0; |
|
310 |
zpull(&strm, in); |
|
311 |
start = strm.next_in; |
|
312 |
last = start[0] & 1; |
|
313 |
if (last && clr) |
|
314 |
start[0] &= ~1; |
|
315 |
strm.avail_out = 0; |
|
316 |
for (;;) { |
|
317 |
/* if input used and output done, write used input and get more */
|
|
318 |
if (strm.avail_in == 0 && strm.avail_out != 0) { |
|
319 |
fwrite(start, 1, strm.next_in - start, out); |
|
320 |
start = in->buf; |
|
321 |
in->left = 0; |
|
322 |
zpull(&strm, in); |
|
323 |
}
|
|
324 |
||
325 |
/* decompress -- return early when end-of-block reached */
|
|
326 |
strm.avail_out = CHUNK; |
|
327 |
strm.next_out = junk; |
|
328 |
ret = inflate(&strm, Z_BLOCK); |
|
329 |
switch (ret) { |
|
330 |
case Z_MEM_ERROR: |
|
331 |
bail("out of memory", ""); |
|
332 |
case Z_DATA_ERROR: |
|
333 |
bail("invalid compressed data in ", in->name); |
|
334 |
}
|
|
335 |
||
336 |
/* update length of uncompressed data */
|
|
337 |
len += CHUNK - strm.avail_out; |
|
338 |
||
339 |
/* check for block boundary (only get this when block copied out) */
|
|
340 |
if (strm.data_type & 128) { |
|
341 |
/* if that was the last block, then done */
|
|
342 |
if (last) |
|
343 |
break; |
|
344 |
||
345 |
/* number of unused bits in last byte */
|
|
346 |
pos = strm.data_type & 7; |
|
347 |
||
348 |
/* find the next last-block bit */
|
|
349 |
if (pos != 0) { |
|
350 |
/* next last-block bit is in last used byte */
|
|
351 |
pos = 0x100 >> pos; |
|
352 |
last = strm.next_in[-1] & pos; |
|
353 |
if (last && clr) |
|
354 |
strm.next_in[-1] &= ~pos; |
|
355 |
}
|
|
356 |
else { |
|
357 |
/* next last-block bit is in next unused byte */
|
|
358 |
if (strm.avail_in == 0) { |
|
359 |
/* don't have that byte yet -- get it */
|
|
360 |
fwrite(start, 1, strm.next_in - start, out); |
|
361 |
start = in->buf; |
|
362 |
in->left = 0; |
|
363 |
zpull(&strm, in); |
|
364 |
}
|
|
365 |
last = strm.next_in[0] & 1; |
|
366 |
if (last && clr) |
|
367 |
strm.next_in[0] &= ~1; |
|
368 |
}
|
|
369 |
}
|
|
370 |
}
|
|
371 |
||
372 |
/* update buffer with unused input */
|
|
373 |
in->left = strm.avail_in; |
|
374 |
in->next = strm.next_in; |
|
375 |
||
376 |
/* copy used input, write empty blocks to get to byte boundary */
|
|
377 |
pos = strm.data_type & 7; |
|
378 |
fwrite(start, 1, in->next - start - 1, out); |
|
379 |
last = in->next[-1]; |
|
380 |
if (pos == 0 || !clr) |
|
381 |
/* already at byte boundary, or last file: write last byte */
|
|
382 |
putc(last, out); |
|
383 |
else { |
|
384 |
/* append empty blocks to last byte */
|
|
385 |
last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */ |
|
386 |
if (pos & 1) { |
|
387 |
/* odd -- append an empty stored block */
|
|
388 |
putc(last, out); |
|
389 |
if (pos == 1) |
|
390 |
putc(0, out); /* two more bits in block header */ |
|
391 |
fwrite("\0\0\xff\xff", 1, 4, out); |
|
392 |
}
|
|
393 |
else { |
|
394 |
/* even -- append 1, 2, or 3 empty fixed blocks */
|
|
395 |
switch (pos) { |
|
396 |
case 6: |
|
397 |
putc(last | 8, out); |
|
398 |
last = 0; |
|
399 |
case 4: |
|
400 |
putc(last | 0x20, out); |
|
401 |
last = 0; |
|
402 |
case 2: |
|
403 |
putc(last | 0x80, out); |
|
404 |
putc(0, out); |
|
405 |
}
|
|
406 |
}
|
|
407 |
}
|
|
408 |
||
409 |
/* update crc and tot */
|
|
410 |
*crc = crc32_combine(*crc, bget4(in), len); |
|
411 |
*tot += (unsigned long)len; |
|
412 |
||
413 |
/* clean up */
|
|
414 |
inflateEnd(&strm); |
|
415 |
free(junk); |
|
416 |
bclose(in); |
|
417 |
||
418 |
/* write trailer if this is the last gzip file */
|
|
419 |
if (!clr) { |
|
420 |
put4(*crc, out); |
|
421 |
put4(*tot, out); |
|
422 |
}
|
|
423 |
}
|
|
424 |
||
425 |
/* join the gzip files on the command line, write result to stdout */
|
|
426 |
int main(int argc, char **argv) |
|
427 |
{
|
|
428 |
unsigned long crc, tot; /* running crc and total uncompressed length */ |
|
429 |
||
430 |
/* skip command name */
|
|
431 |
argc--; |
|
432 |
argv++; |
|
433 |
||
434 |
/* show usage if no arguments */
|
|
435 |
if (argc == 0) { |
|
436 |
fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n", |
|
437 |
stderr); |
|
438 |
return 0; |
|
439 |
}
|
|
440 |
||
441 |
/* join gzip files on command line and write to stdout */
|
|
442 |
gzinit(&crc, &tot, stdout); |
|
443 |
while (argc--) |
|
444 |
gzcopy(*argv++, argc, &crc, &tot, stdout); |
|
445 |
||
446 |
/* done */
|
|
447 |
return 0; |
|
448 |
}
|