~akopytov/percona-xtrabackup/bug1166888-2.0

Viewing changes to src/libarchive/libarchive/archive_write_set_compression_bzip2.c

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

/*-

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR

* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "archive_platform.h"

__FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_compression_bzip2.c 201091 2009-12-28 02:22:41Z kientzle $");

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#include <stdio.h>

#ifdef HAVE_STDLIB_H

#include <stdlib.h>

#endif

#ifdef HAVE_STRING_H

#include <string.h>

#endif

#ifdef HAVE_BZLIB_H

#include <bzlib.h>

#endif

#include "archive.h"

#include "archive_private.h"

#include "archive_write_private.h"

#if !defined(HAVE_BZLIB_H) || !defined(BZ_CONFIG_ERROR)

int

archive_write_set_compression_bzip2(struct archive *a)

{

archive_set_error(a, ARCHIVE_ERRNO_MISC,

"bzip2 compression not supported on this platform");

return (ARCHIVE_FATAL);

}

#else

/* Don't compile this if we don't have bzlib. */

struct private_data {

bz_stream stream;

int64_t total_in;

char *compressed;

size_t compressed_buffer_size;

};

struct private_config {

int compression_level;

};

* Yuck. bzlib.h is not const-correct, so I need this one bit

* of ugly hackery to convert a const * pointer to a non-const pointer.

#define SET_NEXT_IN(st,src) \

(st)->stream.next_in = (char *)(uintptr_t)(const void *)(src)

static int archive_compressor_bzip2_finish(struct archive_write *);

static int archive_compressor_bzip2_init(struct archive_write *);

static int archive_compressor_bzip2_options(struct archive_write *,

const char *, const char *);

static int archive_compressor_bzip2_write(struct archive_write *,

const void *, size_t);

static int drive_compressor(struct archive_write *, struct private_data *,

int finishing);

* Allocate, initialize and return an archive object.

int

archive_write_set_compression_bzip2(struct archive *_a)

{

struct archive_write *a = (struct archive_write *)_a;

struct private_config *config;

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

ARCHIVE_STATE_NEW, "archive_write_set_compression_bzip2");

config = malloc(sizeof(*config));

if (config == NULL) {

archive_set_error(&a->archive, ENOMEM, "Out of memory");

return (ARCHIVE_FATAL);

100

}

101

a->compressor.config = config;

102

a->compressor.finish = archive_compressor_bzip2_finish;

103

config->compression_level = 9; /* default */

104

a->compressor.init = &archive_compressor_bzip2_init;

105

a->compressor.options = &archive_compressor_bzip2_options;

106

a->archive.compression_code = ARCHIVE_COMPRESSION_BZIP2;

107

a->archive.compression_name = "bzip2";

108

return (ARCHIVE_OK);

109

}

110

111

112

* Setup callback.

113

114

static int

115

archive_compressor_bzip2_init(struct archive_write *a)

116

{

117

int ret;

118

struct private_data *state;

119

struct private_config *config;

120

121

config = (struct private_config *)a->compressor.config;

122

if (a->client_opener != NULL) {

123

ret = (a->client_opener)(&a->archive, a->client_data);

124

if (ret != 0)

125

return (ret);

126

}

127

128

state = (struct private_data *)malloc(sizeof(*state));

129

if (state == NULL) {

130

archive_set_error(&a->archive, ENOMEM,

131

"Can't allocate data for compression");

132

return (ARCHIVE_FATAL);

133

}

134

memset(state, 0, sizeof(*state));

135

136

state->compressed_buffer_size = a->bytes_per_block;

137

state->compressed = (char *)malloc(state->compressed_buffer_size);

138

139

if (state->compressed == NULL) {

140

archive_set_error(&a->archive, ENOMEM,

141

"Can't allocate data for compression buffer");

142

free(state);

143

return (ARCHIVE_FATAL);

144

}

145

146

state->stream.next_out = state->compressed;

147

state->stream.avail_out = state->compressed_buffer_size;

148

a->compressor.write = archive_compressor_bzip2_write;

149

150

/* Initialize compression library */

151

ret = BZ2_bzCompressInit(&(state->stream),

152

config->compression_level, 0, 30);

153

if (ret == BZ_OK) {

154

a->compressor.data = state;

155

return (ARCHIVE_OK);

156

}

157

158

/* Library setup failed: clean up. */

159

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

160

"Internal error initializing compression library");

161

free(state->compressed);

162

free(state);

163

164

/* Override the error message if we know what really went wrong. */

165

switch (ret) {

166

case BZ_PARAM_ERROR:

167

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

168

"Internal error initializing compression library: "

169

"invalid setup parameter");

170

break;

171

case BZ_MEM_ERROR:

172

archive_set_error(&a->archive, ENOMEM,

173

"Internal error initializing compression library: "

174

"out of memory");

175

break;

176

case BZ_CONFIG_ERROR:

177

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

178

"Internal error initializing compression library: "

179

"mis-compiled library");

180

break;

181

}

182

183

return (ARCHIVE_FATAL);

184

185

}

186

187

188

* Set write options.

189

190

static int

191

archive_compressor_bzip2_options(struct archive_write *a, const char *key,

192

const char *value)

193

{

194

struct private_config *config;

195

196

config = (struct private_config *)a->compressor.config;

197

if (strcmp(key, "compression-level") == 0) {

198

if (value == NULL || !(value[0] >= '0' && value[0] <= '9') ||

199

value[1] != '\0')

200

return (ARCHIVE_WARN);

201

config->compression_level = value[0] - '0';

202

/* Make '0' be a synonym for '1'. */

203

/* This way, bzip2 compressor supports the same 0..9

204

* range of levels as gzip. */

205

if (config->compression_level < 1)

206

config->compression_level = 1;

207

return (ARCHIVE_OK);

208

}

209

210

return (ARCHIVE_WARN);

211

}

212

213

214

* Write data to the compressed stream.

215

216

* Returns ARCHIVE_OK if all data written, error otherwise.

217

218

static int

219

archive_compressor_bzip2_write(struct archive_write *a, const void *buff,

220

size_t length)

221

{

222

struct private_data *state;

223

224

state = (struct private_data *)a->compressor.data;

225

if (a->client_writer == NULL) {

226

archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER,

227

"No write callback is registered? "

228

"This is probably an internal programming error.");

229

return (ARCHIVE_FATAL);

230

}

231

232

/* Update statistics */

233

state->total_in += length;

234

235

/* Compress input data to output buffer */

236

SET_NEXT_IN(state, buff);

237

state->stream.avail_in = length;

238

if (drive_compressor(a, state, 0))

239

return (ARCHIVE_FATAL);

240

a->archive.file_position += length;

241

return (ARCHIVE_OK);

242

}

243

244

245

246

* Finish the compression.

247

248

static int

249

archive_compressor_bzip2_finish(struct archive_write *a)

250

{

251

ssize_t block_length;

252

int ret;

253

struct private_data *state;

254

ssize_t target_block_length;

255

ssize_t bytes_written;

256

unsigned tocopy;

257

258

ret = ARCHIVE_OK;

259

state = (struct private_data *)a->compressor.data;

260

if (state != NULL) {

261

if (a->client_writer == NULL) {

262

archive_set_error(&a->archive,

263

ARCHIVE_ERRNO_PROGRAMMER,

264

"No write callback is registered?\n"

265

"This is probably an internal programming error.");

266

ret = ARCHIVE_FATAL;

267

goto cleanup;

268

}

269

270

/* By default, always pad the uncompressed data. */

271

if (a->pad_uncompressed) {

272

tocopy = a->bytes_per_block -

273

(state->total_in % a->bytes_per_block);

274

while (tocopy > 0 && tocopy < (unsigned)a->bytes_per_block) {

275

SET_NEXT_IN(state, a->nulls);

276

state->stream.avail_in = tocopy < a->null_length ?

277

tocopy : a->null_length;

278

state->total_in += state->stream.avail_in;

279

tocopy -= state->stream.avail_in;

280

ret = drive_compressor(a, state, 0);

281

if (ret != ARCHIVE_OK)

282

goto cleanup;

283

}

284

}

285

286

/* Finish compression cycle. */

287

if ((ret = drive_compressor(a, state, 1)))

288

goto cleanup;

289

290

/* Optionally, pad the final compressed block. */

291

block_length = state->stream.next_out - state->compressed;

292

293

/* Tricky calculation to determine size of last block. */

294

if (a->bytes_in_last_block <= 0)

295

/* Default or Zero: pad to full block */

296

target_block_length = a->bytes_per_block;

297

else

298

/* Round length to next multiple of bytes_in_last_block. */

299

target_block_length = a->bytes_in_last_block *

300

( (block_length + a->bytes_in_last_block - 1) /

301

a->bytes_in_last_block);

302

if (target_block_length > a->bytes_per_block)

303

target_block_length = a->bytes_per_block;

304

if (block_length < target_block_length) {

305

memset(state->stream.next_out, 0,

306

target_block_length - block_length);

307

block_length = target_block_length;

308

}

309

310

/* Write the last block */

311

bytes_written = (a->client_writer)(&a->archive, a->client_data,

312

state->compressed, block_length);

313

314

/* TODO: Handle short write of final block. */

315

if (bytes_written <= 0)

316

ret = ARCHIVE_FATAL;

317

else {

318

a->archive.raw_position += ret;

319

ret = ARCHIVE_OK;

320

}

321

322

/* Cleanup: shut down compressor, release memory, etc. */

323

cleanup:

324

switch (BZ2_bzCompressEnd(&(state->stream))) {

325

case BZ_OK:

326

break;

327

default:

328

archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER,

329

"Failed to clean up compressor");

330

ret = ARCHIVE_FATAL;

331

}

332

333

free(state->compressed);

334

free(state);

335

}

336

/* Free configuration data even if we were never fully initialized. */

337

free(a->compressor.config);

338

a->compressor.config = NULL;

339

return (ret);

340

}

341

342

343

* Utility function to push input data through compressor, writing

344

* full output blocks as necessary.

345

346

* Note that this handles both the regular write case (finishing ==

347

* false) and the end-of-archive case (finishing == true).

348

349

static int

350

drive_compressor(struct archive_write *a, struct private_data *state, int finishing)

351

{

352

ssize_t bytes_written;

353

int ret;

354

355

for (;;) {

356

if (state->stream.avail_out == 0) {

357

bytes_written = (a->client_writer)(&a->archive,

358

a->client_data, state->compressed,

359

state->compressed_buffer_size);

360

if (bytes_written <= 0) {

361

/* TODO: Handle this write failure */

362

return (ARCHIVE_FATAL);

363

} else if ((size_t)bytes_written < state->compressed_buffer_size) {

364

/* Short write: Move remainder to

365

* front and keep filling */

366

memmove(state->compressed,

367

state->compressed + bytes_written,

368

state->compressed_buffer_size - bytes_written);

369

}

370

371

a->archive.raw_position += bytes_written;

372

state->stream.next_out = state->compressed +

373

state->compressed_buffer_size - bytes_written;

374

state->stream.avail_out = bytes_written;

375

}

376

377

/* If there's nothing to do, we're done. */

378

if (!finishing && state->stream.avail_in == 0)

379

return (ARCHIVE_OK);

380

381

ret = BZ2_bzCompress(&(state->stream),

382

finishing ? BZ_FINISH : BZ_RUN);

383

384

switch (ret) {

385

case BZ_RUN_OK:

386

/* In non-finishing case, did compressor

387

* consume everything? */

388

if (!finishing && state->stream.avail_in == 0)

389

return (ARCHIVE_OK);

390

break;

391

case BZ_FINISH_OK: /* Finishing: There's more work to do */

392

break;

393

case BZ_STREAM_END: /* Finishing: all done */

394

/* Only occurs in finishing case */

395

return (ARCHIVE_OK);

396

default:

397

/* Any other return value indicates an error */

398

archive_set_error(&a->archive,

399

ARCHIVE_ERRNO_PROGRAMMER,

400

"Bzip2 compression failed;"

401

" BZ2_bzCompress() returned %d",

402

ret);

403

return (ARCHIVE_FATAL);

404

}

405

}

406

}

407

408

#endif /* HAVE_BZLIB_H && BZ_CONFIG_ERROR */

Older »