~akopytov/percona-xtrabackup/bug1166888-2.1

Viewing changes to src/libarchive/libarchive/archive_write_set_compression_gzip.c

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: (391.1.5 staging)
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/libarchive/libarchive/archive_write_set_compression_gzip.c

/*-

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR

* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "archive_platform.h"

__FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_compression_gzip.c 201081 2009-12-28 02:04:42Z kientzle $");

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#ifdef HAVE_STDLIB_H

#include <stdlib.h>

#endif

#ifdef HAVE_STRING_H

#include <string.h>

#endif

#include <time.h>

#ifdef HAVE_ZLIB_H

#include <zlib.h>

#endif

#include "archive.h"

#include "archive_private.h"

#include "archive_write_private.h"

#ifndef HAVE_ZLIB_H

int

archive_write_set_compression_gzip(struct archive *a)

{

archive_set_error(a, ARCHIVE_ERRNO_MISC,

"gzip compression not supported on this platform");

return (ARCHIVE_FATAL);

}

#else

/* Don't compile this if we don't have zlib. */

struct private_data {

z_stream stream;

int64_t total_in;

unsigned char *compressed;

size_t compressed_buffer_size;

unsigned long crc;

};

struct private_config {

int compression_level;

};

* Yuck. zlib.h is not const-correct, so I need this one bit

* of ugly hackery to convert a const * pointer to a non-const pointer.

#define SET_NEXT_IN(st,src) \

(st)->stream.next_in = (Bytef *)(uintptr_t)(const void *)(src)

static int archive_compressor_gzip_finish(struct archive_write *);

static int archive_compressor_gzip_init(struct archive_write *);

static int archive_compressor_gzip_options(struct archive_write *,

const char *, const char *);

static int archive_compressor_gzip_write(struct archive_write *,

const void *, size_t);

static int drive_compressor(struct archive_write *, struct private_data *,

int finishing);

* Allocate, initialize and return a archive object.

int

archive_write_set_compression_gzip(struct archive *_a)

{

struct archive_write *a = (struct archive_write *)_a;

struct private_config *config;

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

ARCHIVE_STATE_NEW, "archive_write_set_compression_gzip");

config = malloc(sizeof(*config));

100

if (config == NULL) {

101

archive_set_error(&a->archive, ENOMEM, "Out of memory");

102

return (ARCHIVE_FATAL);

103

}

104

a->compressor.config = config;

105

a->compressor.finish = &archive_compressor_gzip_finish;

106

config->compression_level = Z_DEFAULT_COMPRESSION;

107

a->compressor.init = &archive_compressor_gzip_init;

108

a->compressor.options = &archive_compressor_gzip_options;

109

a->archive.compression_code = ARCHIVE_COMPRESSION_GZIP;

110

a->archive.compression_name = "gzip";

111

return (ARCHIVE_OK);

112

}

113

114

115

* Setup callback.

116

117

static int

118

archive_compressor_gzip_init(struct archive_write *a)

119

{

120

int ret;

121

struct private_data *state;

122

struct private_config *config;

123

time_t t;

124

125

config = (struct private_config *)a->compressor.config;

126

127

if (a->client_opener != NULL) {

128

ret = (a->client_opener)(&a->archive, a->client_data);

129

if (ret != ARCHIVE_OK)

130

return (ret);

131

}

132

133

134

* The next check is a temporary workaround until the gzip

135

* code can be overhauled some. The code should not require

136

* that compressed_buffer_size == bytes_per_block. Removing

137

* this assumption will allow us to compress larger chunks at

138

* a time, which should improve overall performance

139

* marginally. As a minor side-effect, such a cleanup would

140

* allow us to support truly arbitrary block sizes.

141

142

if (a->bytes_per_block < 10) {

143

archive_set_error(&a->archive, EINVAL,

144

"GZip compressor requires a minimum 10 byte block size");

145

return (ARCHIVE_FATAL);

146

}

147

148

state = (struct private_data *)malloc(sizeof(*state));

149

if (state == NULL) {

150

archive_set_error(&a->archive, ENOMEM,

151

"Can't allocate data for compression");

152

return (ARCHIVE_FATAL);

153

}

154

memset(state, 0, sizeof(*state));

155

156

157

* See comment above. We should set compressed_buffer_size to

158

* max(bytes_per_block, 65536), but the code can't handle that yet.

159

160

state->compressed_buffer_size = a->bytes_per_block;

161

state->compressed = (unsigned char *)malloc(state->compressed_buffer_size);

162

state->crc = crc32(0L, NULL, 0);

163

164

if (state->compressed == NULL) {

165

archive_set_error(&a->archive, ENOMEM,

166

"Can't allocate data for compression buffer");

167

free(state);

168

return (ARCHIVE_FATAL);

169

}

170

171

state->stream.next_out = state->compressed;

172

state->stream.avail_out = state->compressed_buffer_size;

173

174

/* Prime output buffer with a gzip header. */

175

t = time(NULL);

176

state->compressed[0] = 0x1f; /* GZip signature bytes */

177

state->compressed[1] = 0x8b;

178

state->compressed[2] = 0x08; /* "Deflate" compression */

179

state->compressed[3] = 0; /* No options */

180

state->compressed[4] = (t)&0xff; /* Timestamp */

181

state->compressed[5] = (t>>8)&0xff;

182

state->compressed[6] = (t>>16)&0xff;

183

state->compressed[7] = (t>>24)&0xff;

184

state->compressed[8] = 0; /* No deflate options */

185

state->compressed[9] = 3; /* OS=Unix */

186

state->stream.next_out += 10;

187

state->stream.avail_out -= 10;

188

189

a->compressor.write = archive_compressor_gzip_write;

190

191

/* Initialize compression library. */

192

ret = deflateInit2(&(state->stream),

193

config->compression_level,

194

Z_DEFLATED,

195

-15 /* < 0 to suppress zlib header */,

196

197

Z_DEFAULT_STRATEGY);

198

199

if (ret == Z_OK) {

200

a->compressor.data = state;

201

return (0);

202

}

203

204

/* Library setup failed: clean up. */

205

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Internal error "

206

"initializing compression library");

207

free(state->compressed);

208

free(state);

209

210

/* Override the error message if we know what really went wrong. */

211

switch (ret) {

212

case Z_STREAM_ERROR:

213

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

214

"Internal error initializing "

215

"compression library: invalid setup parameter");

216

break;

217

case Z_MEM_ERROR:

218

archive_set_error(&a->archive, ENOMEM, "Internal error initializing "

219

"compression library");

220

break;

221

case Z_VERSION_ERROR:

222

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

223

"Internal error initializing "

224

"compression library: invalid library version");

225

break;

226

}

227

228

return (ARCHIVE_FATAL);

229

}

230

231

232

* Set write options.

233

234

static int

235

archive_compressor_gzip_options(struct archive_write *a, const char *key,

236

const char *value)

237

{

238

struct private_config *config;

239

240

config = (struct private_config *)a->compressor.config;

241

if (strcmp(key, "compression-level") == 0) {

242

if (value == NULL || !(value[0] >= '0' && value[0] <= '9') ||

243

value[1] != '\0')

244

return (ARCHIVE_WARN);

245

config->compression_level = value[0] - '0';

246

return (ARCHIVE_OK);

247

}

248

249

return (ARCHIVE_WARN);

250

}

251

252

253

* Write data to the compressed stream.

254

255

static int

256

archive_compressor_gzip_write(struct archive_write *a, const void *buff,

257

size_t length)

258

{

259

struct private_data *state;

260

int ret;

261

262

state = (struct private_data *)a->compressor.data;

263

if (a->client_writer == NULL) {

264

archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER,

265

"No write callback is registered? "

266

"This is probably an internal programming error.");

267

return (ARCHIVE_FATAL);

268

}

269

270

/* Update statistics */

271

state->crc = crc32(state->crc, (const Bytef *)buff, length);

272

state->total_in += length;

273

274

/* Compress input data to output buffer */

275

SET_NEXT_IN(state, buff);

276

state->stream.avail_in = length;

277

if ((ret = drive_compressor(a, state, 0)) != ARCHIVE_OK)

278

return (ret);

279

280

a->archive.file_position += length;

281

return (ARCHIVE_OK);

282

}

283

284

285

* Finish the compression...

286

287

static int

288

archive_compressor_gzip_finish(struct archive_write *a)

289

{

290

ssize_t block_length, target_block_length, bytes_written;

291

int ret;

292

struct private_data *state;

293

unsigned tocopy;

294

unsigned char trailer[8];

295

296

state = (struct private_data *)a->compressor.data;

297

ret = 0;

298

if (state != NULL) {

299

if (a->client_writer == NULL) {

300

archive_set_error(&a->archive,

301

ARCHIVE_ERRNO_PROGRAMMER,

302

"No write callback is registered? "

303

"This is probably an internal programming error.");

304

ret = ARCHIVE_FATAL;

305

goto cleanup;

306

}

307

308

/* By default, always pad the uncompressed data. */

309

if (a->pad_uncompressed) {

310

tocopy = a->bytes_per_block -

311

(state->total_in % a->bytes_per_block);

312

while (tocopy > 0 && tocopy < (unsigned)a->bytes_per_block) {

313

SET_NEXT_IN(state, a->nulls);

314

state->stream.avail_in = tocopy < a->null_length ?

315

tocopy : a->null_length;

316

state->crc = crc32(state->crc, a->nulls,

317

state->stream.avail_in);

318

state->total_in += state->stream.avail_in;

319

tocopy -= state->stream.avail_in;

320

ret = drive_compressor(a, state, 0);

321

if (ret != ARCHIVE_OK)

322

goto cleanup;

323

}

324

}

325

326

/* Finish compression cycle */

327

if (((ret = drive_compressor(a, state, 1))) != ARCHIVE_OK)

328

goto cleanup;

329

330

/* Build trailer: 4-byte CRC and 4-byte length. */

331

trailer[0] = (state->crc)&0xff;

332

trailer[1] = (state->crc >> 8)&0xff;

333

trailer[2] = (state->crc >> 16)&0xff;

334

trailer[3] = (state->crc >> 24)&0xff;

335

trailer[4] = (state->total_in)&0xff;

336

trailer[5] = (state->total_in >> 8)&0xff;

337

trailer[6] = (state->total_in >> 16)&0xff;

338

trailer[7] = (state->total_in >> 24)&0xff;

339

340

/* Add trailer to current block. */

341

tocopy = 8;

342

if (tocopy > state->stream.avail_out)

343

tocopy = state->stream.avail_out;

344

memcpy(state->stream.next_out, trailer, tocopy);

345

state->stream.next_out += tocopy;

346

state->stream.avail_out -= tocopy;

347

348

/* If it overflowed, flush and start a new block. */

349

if (tocopy < 8) {

350

bytes_written = (a->client_writer)(&a->archive, a->client_data,

351

state->compressed, state->compressed_buffer_size);

352

if (bytes_written <= 0) {

353

ret = ARCHIVE_FATAL;

354

goto cleanup;

355

}

356

a->archive.raw_position += bytes_written;

357

state->stream.next_out = state->compressed;

358

state->stream.avail_out = state->compressed_buffer_size;

359

memcpy(state->stream.next_out, trailer + tocopy, 8-tocopy);

360

state->stream.next_out += 8-tocopy;

361

state->stream.avail_out -= 8-tocopy;

362

}

363

364

/* Optionally, pad the final compressed block. */

365

block_length = state->stream.next_out - state->compressed;

366

367

/* Tricky calculation to determine size of last block. */

368

if (a->bytes_in_last_block <= 0)

369

/* Default or Zero: pad to full block */

370

target_block_length = a->bytes_per_block;

371

else

372

/* Round length to next multiple of bytes_in_last_block. */

373

target_block_length = a->bytes_in_last_block *

374

( (block_length + a->bytes_in_last_block - 1) /

375

a->bytes_in_last_block);

376

if (target_block_length > a->bytes_per_block)

377

target_block_length = a->bytes_per_block;

378

if (block_length < target_block_length) {

379

memset(state->stream.next_out, 0,

380

target_block_length - block_length);

381

block_length = target_block_length;

382

}

383

384

/* Write the last block */

385

bytes_written = (a->client_writer)(&a->archive, a->client_data,

386

state->compressed, block_length);

387

if (bytes_written <= 0) {

388

ret = ARCHIVE_FATAL;

389

goto cleanup;

390

}

391

a->archive.raw_position += bytes_written;

392

393

/* Cleanup: shut down compressor, release memory, etc. */

394

cleanup:

395

switch (deflateEnd(&(state->stream))) {

396

case Z_OK:

397

break;

398

default:

399

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

400

"Failed to clean up compressor");

401

ret = ARCHIVE_FATAL;

402

}

403

free(state->compressed);

404

free(state);

405

}

406

/* Clean up config area even if we never initialized. */

407

free(a->compressor.config);

408

a->compressor.config = NULL;

409

return (ret);

410

}

411

412

413

* Utility function to push input data through compressor,

414

* writing full output blocks as necessary.

415

416

* Note that this handles both the regular write case (finishing ==

417

* false) and the end-of-archive case (finishing == true).

418

419

static int

420

drive_compressor(struct archive_write *a, struct private_data *state, int finishing)

421

{

422

ssize_t bytes_written;

423

int ret;

424

425

for (;;) {

426

if (state->stream.avail_out == 0) {

427

bytes_written = (a->client_writer)(&a->archive,

428

a->client_data, state->compressed,

429

state->compressed_buffer_size);

430

if (bytes_written <= 0) {

431

/* TODO: Handle this write failure */

432

return (ARCHIVE_FATAL);

433

} else if ((size_t)bytes_written < state->compressed_buffer_size) {

434

/* Short write: Move remaining to

435

* front of block and keep filling */

436

memmove(state->compressed,

437

state->compressed + bytes_written,

438

state->compressed_buffer_size - bytes_written);

439

}

440

a->archive.raw_position += bytes_written;

441

state->stream.next_out

442

= state->compressed +

443

state->compressed_buffer_size - bytes_written;

444

state->stream.avail_out = bytes_written;

445

}

446

447

/* If there's nothing to do, we're done. */

448

if (!finishing && state->stream.avail_in == 0)

449

return (ARCHIVE_OK);

450

451

ret = deflate(&(state->stream),

452

finishing ? Z_FINISH : Z_NO_FLUSH );

453

454

switch (ret) {

455

case Z_OK:

456

/* In non-finishing case, check if compressor

457

* consumed everything */

458

if (!finishing && state->stream.avail_in == 0)

459

return (ARCHIVE_OK);

460

/* In finishing case, this return always means

461

* there's more work */

462

break;

463

case Z_STREAM_END:

464

/* This return can only occur in finishing case. */

465

return (ARCHIVE_OK);

466

default:

467

/* Any other return value indicates an error. */

468

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

469

"GZip compression failed:"

470

" deflate() call returned status %d",

471

ret);

472

return (ARCHIVE_FATAL);

473

}

474

}

475

}

476

477

#endif /* HAVE_ZLIB_H */

Older »