~akopytov/percona-xtrabackup/bug1166888-2.0

Viewing changes to src/libarchive/libarchive/archive_read_support_compression_gzip.c

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/libarchive/libarchive/archive_read_support_compression_gzip.c

/*-

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR

* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "archive_platform.h"

__FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_compression_gzip.c 201082 2009-12-28 02:05:28Z kientzle $");

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#ifdef HAVE_STDLIB_H

#include <stdlib.h>

#endif

#ifdef HAVE_STRING_H

#include <string.h>

#endif

#ifdef HAVE_UNISTD_H

#include <unistd.h>

#endif

#ifdef HAVE_ZLIB_H

#include <zlib.h>

#endif

#include "archive.h"

#include "archive_private.h"

#include "archive_read_private.h"

#ifdef HAVE_ZLIB_H

struct private_data {

z_stream stream;

char in_stream;

unsigned char *out_block;

size_t out_block_size;

int64_t total_out;

unsigned long crc;

char eof; /* True = found end of compressed data. */

};

/* Gzip Filter. */

static ssize_t gzip_filter_read(struct archive_read_filter *, const void **);

static int gzip_filter_close(struct archive_read_filter *);

#endif

* Note that we can detect gzip archives even if we can't decompress

* them. (In fact, we like detecting them because we can give better

* error messages.) So the bid framework here gets compiled even

* if zlib is unavailable.

* TODO: If zlib is unavailable, gzip_bidder_init() should

* use the compress_program framework to try to fire up an external

* gunzip program.

static int gzip_bidder_bid(struct archive_read_filter_bidder *,

struct archive_read_filter *);

static int gzip_bidder_init(struct archive_read_filter *);

int

archive_read_support_compression_gzip(struct archive *_a)

{

struct archive_read *a = (struct archive_read *)_a;

struct archive_read_filter_bidder *bidder = __archive_read_get_bidder(a);

if (bidder == NULL)

return (ARCHIVE_FATAL);

bidder->data = NULL;

bidder->bid = gzip_bidder_bid;

bidder->init = gzip_bidder_init;

bidder->options = NULL;

bidder->free = NULL; /* No data, so no cleanup necessary. */

/* Signal the extent of gzip support with the return value here. */

#if HAVE_ZLIB_H

return (ARCHIVE_OK);

#else

archive_set_error(_a, ARCHIVE_ERRNO_MISC,

100

"Using external gunzip program");

101

return (ARCHIVE_WARN);

102

#endif

103

}

104

105

106

* Read and verify the header.

107

108

* Returns zero if the header couldn't be validated, else returns

109

* number of bytes in header. If pbits is non-NULL, it receives a

110

* count of bits verified, suitable for use by bidder.

111

112

static int

113

peek_at_header(struct archive_read_filter *filter, int *pbits)

114

{

115

const unsigned char *p;

116

ssize_t avail, len;

117

int bits = 0;

118

int header_flags;

119

120

/* Start by looking at the first ten bytes of the header, which

121

* is all fixed layout. */

122

len = 10;

123

p = __archive_read_filter_ahead(filter, len, &avail);

124

if (p == NULL || avail == 0)

125

return (0);

126

if (p[0] != 037)

127

return (0);

128

bits += 8;

129

if (p[1] != 0213)

130

return (0);

131

bits += 8;

132

if (p[2] != 8) /* We only support deflation. */

133

return (0);

134

bits += 8;

135

if ((p[3] & 0xE0)!= 0) /* No reserved flags set. */

136

return (0);

137

bits += 3;

138

header_flags = p[3];

139

/* Bytes 4-7 are mod time. */

140

/* Byte 8 is deflate flags. */

141

/* XXXX TODO: return deflate flags back to consume_header for use

142

in initializing the decompressor. */

143

/* Byte 9 is OS. */

144

145

/* Optional extra data: 2 byte length plus variable body. */

146

if (header_flags & 4) {

147

p = __archive_read_filter_ahead(filter, len + 2, &avail);

148

if (p == NULL)

149

return (0);

150

len += ((int)p[len + 1] << 8) | (int)p[len];

151

len += 2;

152

}

153

154

/* Null-terminated optional filename. */

155

if (header_flags & 8) {

156

do {

157

++len;

158

if (avail < len)

159

p = __archive_read_filter_ahead(filter,

160

len, &avail);

161

if (p == NULL)

162

return (0);

163

} while (p[len - 1] != 0);

164

}

165

166

/* Null-terminated optional comment. */

167

if (header_flags & 16) {

168

do {

169

++len;

170

if (avail < len)

171

p = __archive_read_filter_ahead(filter,

172

len, &avail);

173

if (p == NULL)

174

return (0);

175

} while (p[len - 1] != 0);

176

}

177

178

/* Optional header CRC */

179

if ((header_flags & 2)) {

180

p = __archive_read_filter_ahead(filter, len + 2, &avail);

181

if (p == NULL)

182

return (0);

183

#if 0

184

int hcrc = ((int)p[len + 1] << 8) | (int)p[len];

185

int crc = /* XXX TODO: Compute header CRC. */;

186

if (crc != hcrc)

187

return (0);

188

bits += 16;

189

#endif

190

len += 2;

191

}

192

193

if (pbits != NULL)

194

*pbits = bits;

195

return (len);

196

}

197

198

199

* Bidder just verifies the header and returns the number of verified bits.

200

201

static int

202

gzip_bidder_bid(struct archive_read_filter_bidder *self,

203

struct archive_read_filter *filter)

204

{

205

int bits_checked;

206

207

(void)self; /* UNUSED */

208

209

if (peek_at_header(filter, &bits_checked))

210

return (bits_checked);

211

return (0);

212

}

213

214

215

#ifndef HAVE_ZLIB_H

216

217

218

* If we don't have the library on this system, we can't do the

219

* decompression directly. We can, however, try to run gunzip

220

* in case that's available.

221

222

static int

223

gzip_bidder_init(struct archive_read_filter *self)

224

{

225

int r;

226

227

r = __archive_read_program(self, "gunzip");

228

/* Note: We set the format here even if __archive_read_program()

229

* above fails. We do, after all, know what the format is

230

* even if we weren't able to read it. */

231

self->code = ARCHIVE_COMPRESSION_GZIP;

232

self->name = "gzip";

233

return (r);

234

}

235

236

#else

237

238

239

* Initialize the filter object.

240

241

static int

242

gzip_bidder_init(struct archive_read_filter *self)

243

{

244

struct private_data *state;

245

static const size_t out_block_size = 64 * 1024;

246

void *out_block;

247

248

self->code = ARCHIVE_COMPRESSION_GZIP;

249

self->name = "gzip";

250

251

state = (struct private_data *)calloc(sizeof(*state), 1);

252

out_block = (unsigned char *)malloc(out_block_size);

253

if (state == NULL || out_block == NULL) {

254

free(out_block);

255

free(state);

256

archive_set_error(&self->archive->archive, ENOMEM,

257

"Can't allocate data for gzip decompression");

258

return (ARCHIVE_FATAL);

259

}

260

261

self->data = state;

262

state->out_block_size = out_block_size;

263

state->out_block = out_block;

264

self->read = gzip_filter_read;

265

self->skip = NULL; /* not supported */

266

self->close = gzip_filter_close;

267

268

state->in_stream = 0; /* We're not actually within a stream yet. */

269

270

return (ARCHIVE_OK);

271

}

272

273

static int

274

consume_header(struct archive_read_filter *self)

275

{

276

struct private_data *state;

277

ssize_t avail;

278

size_t len;

279

int ret;

280

281

state = (struct private_data *)self->data;

282

283

/* If this is a real header, consume it. */

284

len = peek_at_header(self->upstream, NULL);

285

if (len == 0)

286

return (ARCHIVE_EOF);

287

__archive_read_filter_consume(self->upstream, len);

288

289

/* Initialize CRC accumulator. */

290

state->crc = crc32(0L, NULL, 0);

291

292

/* Initialize compression library. */

293

state->stream.next_in = (unsigned char *)(uintptr_t)

294

__archive_read_filter_ahead(self->upstream, 1, &avail);

295

state->stream.avail_in = avail;

296

ret = inflateInit2(&(state->stream),

297

-15 /* Don't check for zlib header */);

298

299

/* Decipher the error code. */

300

switch (ret) {

301

case Z_OK:

302

state->in_stream = 1;

303

return (ARCHIVE_OK);

304

case Z_STREAM_ERROR:

305

archive_set_error(&self->archive->archive,

306

ARCHIVE_ERRNO_MISC,

307

"Internal error initializing compression library: "

308

"invalid setup parameter");

309

break;

310

case Z_MEM_ERROR:

311

archive_set_error(&self->archive->archive, ENOMEM,

312

"Internal error initializing compression library: "

313

"out of memory");

314

break;

315

case Z_VERSION_ERROR:

316

archive_set_error(&self->archive->archive,

317

ARCHIVE_ERRNO_MISC,

318

"Internal error initializing compression library: "

319

"invalid library version");

320

break;

321

default:

322

archive_set_error(&self->archive->archive,

323

ARCHIVE_ERRNO_MISC,

324

"Internal error initializing compression library: "

325

" Zlib error %d", ret);

326

break;

327

}

328

return (ARCHIVE_FATAL);

329

}

330

331

static int

332

consume_trailer(struct archive_read_filter *self)

333

{

334

struct private_data *state;

335

const unsigned char *p;

336

ssize_t avail;

337

338

state = (struct private_data *)self->data;

339

340

state->in_stream = 0;

341

switch (inflateEnd(&(state->stream))) {

342

case Z_OK:

343

break;

344

default:

345

archive_set_error(&self->archive->archive,

346

ARCHIVE_ERRNO_MISC,

347

"Failed to clean up gzip decompressor");

348

return (ARCHIVE_FATAL);

349

}

350

351

/* GZip trailer is a fixed 8 byte structure. */

352

p = __archive_read_filter_ahead(self->upstream, 8, &avail);

353

if (p == NULL || avail == 0)

354

return (ARCHIVE_FATAL);

355

356

/* XXX TODO: Verify the length and CRC. */

357

358

/* We've verified the trailer, so consume it now. */

359

__archive_read_filter_consume(self->upstream, 8);

360

361

return (ARCHIVE_OK);

362

}

363

364

static ssize_t

365

gzip_filter_read(struct archive_read_filter *self, const void **p)

366

{

367

struct private_data *state;

368

size_t decompressed;

369

ssize_t avail_in;

370

int ret;

371

372

state = (struct private_data *)self->data;

373

374

/* Empty our output buffer. */

375

state->stream.next_out = state->out_block;

376

state->stream.avail_out = state->out_block_size;

377

378

/* Try to fill the output buffer. */

379

while (state->stream.avail_out > 0 && !state->eof) {

380

/* If we're not in a stream, read a header

381

* and initialize the decompression library. */

382

if (!state->in_stream) {

383

ret = consume_header(self);

384

if (ret == ARCHIVE_EOF) {

385

state->eof = 1;

386

break;

387

}

388

if (ret < ARCHIVE_OK)

389

return (ret);

390

}

391

392

/* Peek at the next available data. */

393

/* ZLib treats stream.next_in as const but doesn't declare

394

* it so, hence this ugly cast. */

395

state->stream.next_in = (unsigned char *)(uintptr_t)

396

__archive_read_filter_ahead(self->upstream, 1, &avail_in);

397

if (state->stream.next_in == NULL)

398

return (ARCHIVE_FATAL);

399

state->stream.avail_in = avail_in;

400

401

/* Decompress and consume some of that data. */

402

ret = inflate(&(state->stream), 0);

403

switch (ret) {

404

case Z_OK: /* Decompressor made some progress. */

405

__archive_read_filter_consume(self->upstream,

406

avail_in - state->stream.avail_in);

407

break;

408

case Z_STREAM_END: /* Found end of stream. */

409

__archive_read_filter_consume(self->upstream,

410

avail_in - state->stream.avail_in);

411

/* Consume the stream trailer; release the

412

* decompression library. */

413

ret = consume_trailer(self);

414

if (ret < ARCHIVE_OK)

415

return (ret);

416

break;

417

default:

418

/* Return an error. */

419

archive_set_error(&self->archive->archive,

420

ARCHIVE_ERRNO_MISC,

421

"gzip decompression failed");

422

return (ARCHIVE_FATAL);

423

}

424

}

425

426

/* We've read as much as we can. */

427

decompressed = state->stream.next_out - state->out_block;

428

state->total_out += decompressed;

429

if (decompressed == 0)

430

*p = NULL;

431

else

432

*p = state->out_block;

433

return (decompressed);

434

}

435

436

437

* Clean up the decompressor.

438

439

static int

440

gzip_filter_close(struct archive_read_filter *self)

441

{

442

struct private_data *state;

443

int ret;

444

445

state = (struct private_data *)self->data;

446

ret = ARCHIVE_OK;

447

448

if (state->in_stream) {

449

switch (inflateEnd(&(state->stream))) {

450

case Z_OK:

451

break;

452

default:

453

archive_set_error(&(self->archive->archive),

454

ARCHIVE_ERRNO_MISC,

455

"Failed to clean up gzip compressor");

456

ret = ARCHIVE_FATAL;

457

}

458

}

459

460

free(state->out_block);

461

free(state);

462

return (ret);

463

}

464

465

#endif /* HAVE_ZLIB_H */

Older »