1
/* uclpack.c -- example program: a simple file packer
3
This file is part of the UCL data compression library.
5
Copyright (C) 1996-2004 Markus Franz Xaver Johannes Oberhumer
8
The UCL library is free software; you can redistribute it and/or
9
modify it under the terms of the GNU General Public License as
10
published by the Free Software Foundation; either version 2 of
11
the License, or (at your option) any later version.
13
The UCL library is distributed in the hope that it will be useful,
14
but WITHOUT ANY WARRANTY; without even the implied warranty of
15
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
GNU General Public License for more details.
18
You should have received a copy of the GNU General Public License
19
along with the UCL library; see the file COPYING.
20
If not, write to the Free Software Foundation, Inc.,
21
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23
Markus F.X.J. Oberhumer
24
<markus@oberhumer.com>
25
http://www.oberhumer.com/opensource/ucl/
29
/*************************************************************************
30
// NOTE: this is an example program, so do not use to backup your data.
32
// This program lacks things like sophisticated file handling but is
33
// pretty complete regarding compression - it should provide a good
34
// starting point for adaption for you applications.
35
**************************************************************************/
38
#if defined(UCL_USE_ASM)
39
# include <ucl/ucl_asm.h>
41
#if 1 && defined(UCL_USE_ASM)
42
/* use assembler versions of the decompressors */
43
# define ucl_nrv2b_decompress_8 ucl_nrv2b_decompress_asm_8
44
# define ucl_nrv2b_decompress_safe_8 ucl_nrv2b_decompress_asm_safe_8
45
# define ucl_nrv2d_decompress_8 ucl_nrv2d_decompress_asm_8
46
# define ucl_nrv2d_decompress_safe_8 ucl_nrv2d_decompress_asm_safe_8
47
# define ucl_nrv2e_decompress_8 ucl_nrv2e_decompress_asm_8
48
# define ucl_nrv2e_decompress_safe_8 ucl_nrv2e_decompress_asm_safe_8
51
/* benchmark setup - do not introduce floating point dependencies for DOS16 */
52
#if !defined(WITH_TIMER) && !defined(UCL_OS_DOS16)
56
/* portability layer */
57
#define WANT_UCL_MALLOC 1
58
#define WANT_UCL_FREAD 1
59
#if defined(WITH_TIMER)
60
#define WANT_UCL_UCLOCK 1
62
#define WANT_UCL_WILDARGV 1
63
#include "examples/portab.h"
66
static const char *progname = NULL;
68
static unsigned long total_in = 0;
69
static unsigned long total_out = 0;
70
static int opt_debug = 0;
72
/* use option `-F' for faster operation - then we don't compute or verify
73
* a checksum and always use the fast decompressor */
74
static ucl_bool opt_fast = 0;
76
/* magic file header for compressed files */
77
static const unsigned char magic[8] =
78
{ 0x00, 0xe9, 0x55, 0x43, 0x4c, 0xff, 0x01, 0x1a };
81
#if defined(WITH_TIMER)
82
static ucl_uclock_handle_t uc;
83
static double benchmark_bytes;
84
static double benchmark_secs;
88
/*************************************************************************
90
**************************************************************************/
92
ucl_uint xread(FILE *f, ucl_voidp buf, ucl_uint len, ucl_bool allow_eof)
96
l = (ucl_uint) ucl_fread(f,buf,len);
99
fprintf(stderr,"\nsomething's wrong with your C library !!!\n");
102
if (l != len && !allow_eof)
104
fprintf(stderr,"\nread error - premature end of file\n");
111
ucl_uint xwrite(FILE *f, const ucl_voidp buf, ucl_uint len)
117
l = (ucl_uint) ucl_fwrite(f,buf,len);
120
fprintf(stderr,"\nwrite error [%ld %ld] (disk full ?)\n",
133
xread(f,(ucl_voidp) &c,1,0);
137
void xputc(FILE *f, int c)
139
unsigned char cc = (unsigned char) c;
140
xwrite(f,(const ucl_voidp) &cc,1);
143
/* read and write portable 32-bit integers */
145
ucl_uint32 xread32(FILE *f)
151
v = (ucl_uint32) b[3] << 0;
152
v |= (ucl_uint32) b[2] << 8;
153
v |= (ucl_uint32) b[1] << 16;
154
v |= (ucl_uint32) b[0] << 24;
158
void xwrite32(FILE *f, ucl_uint32 v)
162
b[3] = (unsigned char) (v >> 0);
163
b[2] = (unsigned char) (v >> 8);
164
b[1] = (unsigned char) (v >> 16);
165
b[0] = (unsigned char) (v >> 24);
170
/*************************************************************************
172
**************************************************************************/
174
static ucl_uint get_overhead(int method, ucl_uint size)
176
if (method == 0x2b || method == 0x2d || method == 0x2e)
177
return size / 8 + 256;
182
static char method_name[64];
184
static ucl_bool set_method_name(int method, int level)
187
if (level < 1 || level > 10)
190
sprintf(method_name,"NRV2B-99/%d", level);
191
else if (method == 0x2d)
192
sprintf(method_name,"NRV2D-99/%d", level);
193
else if (method == 0x2e)
194
sprintf(method_name,"NRV2E-99/%d", level);
201
/*************************************************************************
203
**************************************************************************/
205
int do_compress(FILE *fi, FILE *fo, int method, int level, ucl_uint block_size)
209
ucl_bytep out = NULL;
212
ucl_uint32 flags = opt_fast ? 0 : 1;
214
ucl_uint overhead = 0;
215
#if defined(WITH_TIMER)
216
ucl_uclock_t t_start, t_stop;
219
total_in = total_out = 0;
222
* Step 1: write magic header, flags & block size, init checksum
224
xwrite(fo,magic,sizeof(magic));
226
xputc(fo,method); /* compression method */
227
xputc(fo,level); /* compression level */
228
xwrite32(fo,block_size);
229
checksum = ucl_adler32(0,NULL,0);
232
* Step 2: allocate compression buffers and work-memory
234
overhead = get_overhead(method,block_size);
235
in = (ucl_bytep) ucl_malloc(block_size);
236
out = (ucl_bytep) ucl_malloc(block_size + overhead);
237
if (in == NULL || out == NULL)
239
printf("%s: out of memory\n", progname);
245
* Step 3: process blocks
250
in_len = xread(fi,in,block_size,1);
254
/* update checksum */
256
checksum = ucl_adler32(checksum,in,in_len);
258
#if defined(WITH_TIMER)
259
ucl_uclock_read(&uc, &t_start);
266
r = ucl_nrv2b_99_compress(in,in_len,out,&out_len,0,level,NULL,NULL);
267
else if (method == 0x2d)
268
r = ucl_nrv2d_99_compress(in,in_len,out,&out_len,0,level,NULL,NULL);
269
else if (method == 0x2e)
270
r = ucl_nrv2e_99_compress(in,in_len,out,&out_len,0,level,NULL,NULL);
271
if (r == UCL_E_OUT_OF_MEMORY)
273
printf("%s: out of memory in compress\n", progname);
277
if (r != UCL_E_OK || out_len > in_len + get_overhead(method,in_len))
279
/* this should NEVER happen */
280
printf("internal error - compression failed: %d\n", r);
285
#if defined(WITH_TIMER)
286
ucl_uclock_read(&uc, &t_stop);
287
benchmark_bytes += in_len;
288
benchmark_secs += ucl_uclock_get_elapsed(&uc, &t_start, &t_stop);
291
/* write uncompressed block size */
294
if (out_len < in_len)
296
/* write compressed block */
297
xwrite32(fo,out_len);
298
xwrite(fo,out,out_len);
302
/* not compressible - write uncompressed block */
304
xwrite(fo,in,in_len);
308
/* write EOF marker */
313
xwrite32(fo,checksum);
323
/*************************************************************************
324
// decompression benchmark
325
**************************************************************************/
327
#if defined(WITH_TIMER)
329
void do_decompress_benchmark(int method, unsigned long benchmark_loops,
330
const ucl_bytep in, ucl_uint in_len,
331
ucl_bytep out, ucl_uint out_len)
333
ucl_uclock_t t_start, t_stop;
335
ucl_uclock_read(&uc, &t_start);
336
while (benchmark_loops-- > 0)
339
ucl_uint new_len = out_len;
341
#if 1 && defined(UCL_USE_ASM)
342
/* use fast assembler versions of the decompressors - see
343
* asm/i386/00README.ASM for more info */
345
r = ucl_nrv2b_decompress_asm_fast_8(in,in_len,out,&new_len,NULL);
346
else if (method == 0x2d)
347
r = ucl_nrv2d_decompress_asm_fast_8(in,in_len,out,&new_len,NULL);
348
else if (method == 0x2e)
349
r = ucl_nrv2e_decompress_asm_fast_8(in,in_len,out,&new_len,NULL);
352
r = ucl_nrv2b_decompress_8(in,in_len,out,&new_len,NULL);
353
else if (method == 0x2d)
354
r = ucl_nrv2d_decompress_8(in,in_len,out,&new_len,NULL);
355
else if (method == 0x2e)
356
r = ucl_nrv2e_decompress_8(in,in_len,out,&new_len,NULL);
358
if (r != UCL_E_OK || new_len != out_len)
360
printf("%s: compressed data violation: error %d (0x%x: %ld/%ld/%ld)\n", progname, r, method, (long) in_len, (long) out_len, (long) new_len);
361
printf("%s: unexpected failure in benchmark -- exiting.\n", progname);
364
benchmark_bytes += out_len;
366
ucl_uclock_read(&uc, &t_stop);
367
benchmark_secs += ucl_uclock_get_elapsed(&uc, &t_start, &t_stop);
370
#endif /* WITH_TIMER */
373
/*************************************************************************
376
// We are using in-place decompression here.
377
**************************************************************************/
379
int do_decompress(FILE *fi, FILE *fo, unsigned long benchmark_loops)
382
ucl_bytep buf = NULL;
384
unsigned char m [ sizeof(magic) ];
390
ucl_uint overhead = 0;
392
total_in = total_out = 0;
395
* Step 1: check magic header, read flags & block size, init checksum
397
if (xread(fi,m,sizeof(magic),1) != sizeof(magic) ||
398
memcmp(m,magic,sizeof(magic)) != 0)
400
printf("%s: header error - this file is not compressed by uclpack\n", progname);
407
block_size = xread32(fi);
408
overhead = get_overhead(method, block_size);
409
if (overhead == 0 || !set_method_name(method, level))
411
printf("%s: header error - invalid method %d (level %d)\n",
412
progname, method, level);
416
if (block_size < 1024 || block_size > 8*1024*1024L)
418
printf("%s: header error - invalid block size %ld\n",
419
progname, (long) block_size);
423
printf("%s: block-size is %ld bytes\n", progname, (long)block_size);
425
checksum = ucl_adler32(0,NULL,0);
428
* Step 2: allocate buffer for in-place decompression
430
buf_len = block_size + overhead;
431
if (benchmark_loops > 0)
433
/* cannot use in-place decompression when doing benchmarks */
434
buf_len += block_size;
436
buf = (ucl_bytep) ucl_malloc(buf_len);
439
printf("%s: out of memory\n", progname);
445
* Step 3: process blocks
454
/* read uncompressed size */
455
out_len = xread32(fi);
457
/* exit if last block (EOF marker) */
461
/* read compressed size */
462
in_len = xread32(fi);
464
/* sanity check of the size values */
465
if (in_len > block_size || out_len > block_size ||
466
in_len == 0 || in_len > out_len)
468
printf("%s: block size error - data corrupted\n", progname);
473
/* place compressed block at the top of the buffer */
474
in = buf + buf_len - in_len;
477
/* read compressed block data */
478
xread(fi,in,in_len,0);
480
if (in_len < out_len)
482
/* decompress - use safe decompressor as data might be corrupted */
483
ucl_uint new_len = out_len;
488
r = ucl_nrv2b_decompress_8(in,in_len,out,&new_len,NULL);
490
r = ucl_nrv2b_decompress_safe_8(in,in_len,out,&new_len,NULL);
492
else if (method == 0x2d)
495
r = ucl_nrv2d_decompress_8(in,in_len,out,&new_len,NULL);
497
r = ucl_nrv2d_decompress_safe_8(in,in_len,out,&new_len,NULL);
499
else if (method == 0x2e)
502
r = ucl_nrv2e_decompress_8(in,in_len,out,&new_len,NULL);
504
r = ucl_nrv2e_decompress_safe_8(in,in_len,out,&new_len,NULL);
506
if (r != UCL_E_OK || new_len != out_len)
508
printf("%s: compressed data violation: error %d (0x%x: %ld/%ld/%ld)\n", progname, r, method, (long) in_len, (long) out_len, (long) new_len);
512
/* write decompressed block */
513
xwrite(fo,out,out_len);
514
/* update checksum */
515
if ((flags & 1) && !opt_fast)
516
checksum = ucl_adler32(checksum,out,out_len);
517
#if defined(WITH_TIMER)
518
if (benchmark_loops > 0)
519
do_decompress_benchmark(method,benchmark_loops,in,in_len,out,out_len);
524
/* write original (incompressible) block */
525
xwrite(fo,in,in_len);
526
/* update checksum */
527
if ((flags & 1) && !opt_fast)
528
checksum = ucl_adler32(checksum,in,in_len);
532
/* read and verify checksum */
535
ucl_uint32 c = xread32(fi);
536
if (!opt_fast && c != checksum)
538
printf("%s: checksum error - data corrupted\n", progname);
551
/*************************************************************************
552
// misc support functions
553
**************************************************************************/
555
static void usage(void)
558
printf(" %s [options] input-file output-file (compress)\n", progname);
559
printf(" %s -d compressed-file output-file (decompress)\n", progname);
560
printf(" %s -t compressed-file... (test)\n", progname);
561
#if defined(WITH_TIMER)
562
printf(" %s -t -D1000 compressed-file... (test decompression speed)\n", progname);
564
printf("\ncompression options:\n");
565
printf(" -1...-9, --10 set compression level [default is `-7']\n");
566
printf(" --nrv2b use NRV2B compression method\n");
567
printf(" --nrv2d use NRV2D compression method [default]\n");
568
printf(" --nrv2e use NRV2E compression method\n");
569
printf("\nother options:\n");
570
printf(" -F do not store or verify a checksum (faster)\n");
571
printf(" -Bxxxx set block-size for compression [default 262144]\n");
572
#if defined(WITH_TIMER)
573
printf(" -Dxxxx number of iterations for decompression benchmark\n");
579
/* open input file */
580
static FILE *xopen_fi(const char *name)
584
f = fopen(name,"rb");
587
printf("%s: cannot open input file %s\n", progname, name);
590
#if defined(HAVE_STAT) && defined(S_ISREG)
593
#if defined(HAVE_LSTAT)
594
if (lstat(name,&st) != 0 || !S_ISREG(st.st_mode))
596
if (stat(name,&st) != 0 || !S_ISREG(st.st_mode))
599
printf("%s: %s is not a regular file\n", progname, name);
609
/* open output file */
610
static FILE *xopen_fo(const char *name)
615
/* this is an example program, so make sure we don't overwrite a file */
616
f = fopen(name,"rb");
619
printf("%s: file %s already exists -- not overwritten\n", progname, name);
624
f = fopen(name,"wb");
627
printf("%s: cannot open output file %s\n", progname, name);
634
/*************************************************************************
636
**************************************************************************/
638
int __acc_cdecl_main main(int argc, char *argv[])
644
const char *in_name = NULL;
645
const char *out_name = NULL;
646
ucl_bool opt_decompress = 0;
647
ucl_bool opt_test = 0;
648
int opt_method = 0x2d;
650
ucl_uint opt_block_size;
651
unsigned long opt_decompress_loops = 0;
654
ucl_wildargv(&argc, &argv);
655
#if defined(WITH_TIMER)
656
ucl_uclock_open(&uc);
659
for (s = progname; *s; s++)
660
if ((*s == '/' || *s == '\\') && s[1])
663
printf("\nUCL data compression library (v%s, %s).\n",
664
ucl_version_string(), ucl_version_date());
665
printf("Copyright (C) 1996-2004 Markus Franz Xaver Johannes Oberhumer\n");
666
printf("http://www.oberhumer.com/opensource/ucl/\n\n");
671
" This is an example program, do not use to backup your data !\n"
677
* Step 1: initialize the UCL library
679
if (ucl_init() != UCL_E_OK)
681
printf("internal error - ucl_init() failed !!!\n");
682
printf("(this usually indicates a compiler bug - try recompiling\nwithout optimizations, and enable `-DUCL_DEBUG' for diagnostics)\n");
688
* Step 2: setup default options
690
opt_block_size = 256 * 1024L;
692
#if defined(ACC_MM_AHSHIFT)
693
/* reduce memory requirements for ancient 640kB DOS real-mode */
694
if (ACC_MM_AHSHIFT != 3)
695
opt_block_size = 16 * 1024L;
700
* Step 3: get options
703
while (i < argc && argv[i][0] == '-')
705
if (strcmp(argv[i],"-d") == 0)
707
else if (strcmp(argv[i],"-t") == 0)
709
else if (strcmp(argv[i],"-F") == 0)
711
else if (strcmp(argv[i],"--2b") == 0)
713
else if (strcmp(argv[i],"--nrv2b") == 0)
715
else if (strcmp(argv[i],"--2d") == 0)
717
else if (strcmp(argv[i],"--nrv2d") == 0)
719
else if (strcmp(argv[i],"--2e") == 0)
721
else if (strcmp(argv[i],"--nrv2e") == 0)
723
else if ((argv[i][1] >= '1' && argv[i][1] <= '9') && !argv[i][2])
724
opt_level = argv[i][1] - '0';
725
else if (strcmp(argv[i],"--10") == 0)
727
else if (strcmp(argv[i],"--best") == 0)
729
else if (argv[i][1] == 'b' && argv[i][2])
731
long x = atol(&argv[i][2]);
732
if (x >= 1024L && x <= 8*1024*1024L)
733
opt_block_size = (ucl_uint) x;
735
printf("%s: error: invalid block-size %ld\n", progname, x);
739
else if (argv[i][1] == 'D' && argv[i][2])
741
#if defined(WITH_TIMER)
742
long x = atol(&argv[i][2]);
744
opt_decompress_loops = x;
746
printf("%s: error: invalid number of benchmark loops %ld\n", progname, x);
753
else if (strcmp(argv[i],"--debug") == 0)
759
if (opt_test && i >= argc)
761
if (!opt_test && i + 2 != argc)
766
* Step 4: process file(s)
771
while (i < argc && r == 0)
774
fi = xopen_fi(in_name);
775
#if defined(WITH_TIMER)
776
benchmark_bytes = benchmark_secs = 0.0;
778
r = do_decompress(fi, NULL, opt_decompress_loops);
780
printf("%s: tested ok: %-10s %-11s: %6lu -> %6lu bytes\n",
781
progname, in_name, method_name, total_in, total_out);
784
#if defined(WITH_TIMER)
785
if (r == 0 && benchmark_bytes > 0)
787
/* speed measured in uncompressed megabytes per second */
788
double mb = benchmark_bytes / (1024.0 * 1024.0);
790
if (benchmark_secs >= 0.0001) mbs = mb / benchmark_secs;
791
printf("\n [benchmark] %lu loops, %.3f MB, %.3f secs, %.3f MB/s\n\n", opt_decompress_loops, mb, benchmark_secs, mbs);
796
else if (opt_decompress)
799
out_name = argv[i++];
800
fi = xopen_fi(in_name);
801
fo = xopen_fo(out_name);
802
r = do_decompress(fi, fo, 0);
804
printf("%s: decompressed %lu into %lu bytes\n",
805
progname, total_in, total_out);
809
printf("%s: using block-size of %lu bytes\n", progname, (unsigned long)opt_block_size);
810
if (!set_method_name(opt_method, opt_level))
812
printf("%s: internal error - invalid method %d (level %d)\n",
813
progname, opt_method, opt_level);
817
out_name = argv[i++];
818
fi = xopen_fi(in_name);
819
fo = xopen_fo(out_name);
820
r = do_compress(fi,fo,opt_method,opt_level,opt_block_size);
823
printf("%s: algorithm %s, compressed %lu into %lu bytes\n",
824
progname, method_name, total_in, total_out);
825
#if defined(WITH_TIMER)
826
if (opt_debug >= 1 && benchmark_bytes > 0)
828
/* speed measured in uncompressed kilobytes per second */
829
double kb = benchmark_bytes / (1024.0);
831
if (benchmark_secs >= 0.0001) kbs = kb / benchmark_secs;
832
printf("\n [compression speeed] %.3f KB, %.3f secs, %.3f KB/s\n\n", kb, benchmark_secs, kbs);
835
#if 1 && defined(WITH_TIMER)
836
printf("\n Info: To test the decompression speed on your system type:\n"
837
" `%s -t -D1000 %s'\n", progname, out_name);
845
#if defined(WITH_TIMER)
846
ucl_uclock_close(&uc);