~akopytov/percona-xtrabackup/bug1166888-2.1

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: (391.1.5 staging)
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/compress.c

/******************************************************

Compressing datasink implementation for XtraBackup.

This program is free software; you can redistribute it and/or modify

it under the terms of the GNU General Public License as published by

the Free Software Foundation; version 2 of the License.

This program is distributed in the hope that it will be useful,

but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

GNU General Public License for more details.

You should have received a copy of the GNU General Public License

along with this program; if not, write to the Free Software

Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

*******************************************************/

#include <my_base.h>

#include <quicklz.h>

#include <univ.i>

#include <zlib.h>

#include "common.h"

#include "datasink.h"

#include "stream.h"

#include "local.h"

#define COMPRESS_CHUNK_SIZE (64 * 1024UL)

#define MY_QLZ_COMPRESS_OVERHEAD 400

typedef struct {

pthread_t id;

uint num;

pthread_mutex_t ctrl_mutex;

pthread_cond_t ctrl_cond;

pthread_mutex_t data_mutex;

pthread_cond_t data_cond;

my_bool started;

my_bool data_avail;

my_bool cancelled;

const char *from;

size_t from_len;

char *to;

size_t to_len;

qlz_state_compress state;

ulong adler;

} comp_thread_ctxt_t;

typedef struct {

ds_ctxt_t *dest_ctxt;

comp_thread_ctxt_t *threads;

uint nthreads;

} ds_compress_ctxt_t;

typedef struct {

datasink_t *dest_ds;

ds_file_t *dest_file;

ds_compress_ctxt_t *comp_ctxt;

size_t bytes_processed;

} ds_compress_file_t;

extern ibool xtrabackup_stream;

extern uint xtrabackup_parallel;

extern ibool xtrabackup_compress_threads;

static ds_ctxt_t *compress_init(const char *root);

static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path,

MY_STAT *mystat);

static int compress_write(ds_file_t *file, const void *buf, size_t len);

static int compress_close(ds_file_t *file);

static void compress_deinit(ds_ctxt_t *ctxt);

datasink_t datasink_compress = {

&compress_init,

&compress_open,

&compress_write,

&compress_close,

&compress_deinit

};

static inline int write_uint32_le(datasink_t *sink, ds_file_t *file,

ulong n);

static inline int write_uint64_le(datasink_t *sink, ds_file_t *file,

ulonglong n);

static comp_thread_ctxt_t *create_worker_threads(uint n);

static void destroy_worker_threads(comp_thread_ctxt_t *threads, uint n);

static void *compress_worker_thread_func(void *arg);

static

ds_ctxt_t *

compress_init(const char *root)

{

ds_ctxt_t *ctxt;

ds_compress_ctxt_t *compress_ctxt;

datasink_t *dest_ds;

100

ds_ctxt_t *dest_ctxt;

101

comp_thread_ctxt_t *threads;

102

103

/* Decide whether the compressed data will be stored in local files or

104

streamed to an archive */

105

dest_ds = xtrabackup_stream ? &datasink_stream : &datasink_local;

106

107

dest_ctxt = dest_ds->init(root);

108

if (dest_ctxt == NULL) {

109

msg("compress: failed to initialize the target datasink.\n");

110

return NULL;

111

}

112

113

/* Create and initialize the worker threads */

114

threads = create_worker_threads(xtrabackup_compress_threads);

115

if (threads == NULL) {

116

msg("compress: failed to create worker threads.\n");

117

dest_ds->deinit(dest_ctxt);

118

return NULL;

119

}

120

121

ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +

122

sizeof(ds_compress_ctxt_t),

123

MYF(MY_FAE));

124

125

compress_ctxt = (ds_compress_ctxt_t *) (ctxt + 1);

126

compress_ctxt->dest_ctxt = dest_ctxt;

127

compress_ctxt->threads = threads;

128

compress_ctxt->nthreads = xtrabackup_compress_threads;

129

130

ctxt->datasink = &datasink_compress;

131

ctxt->ptr = compress_ctxt;

132

133

return ctxt;

134

}

135

136

static

137

ds_file_t *

138

compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)

139

{

140

ds_compress_ctxt_t *comp_ctxt;

141

datasink_t *dest_ds;

142

ds_ctxt_t *dest_ctxt;

143

ds_file_t *dest_file;

144

char new_name[FN_REFLEN];

145

size_t name_len;

146

ds_file_t *file;

147

ds_compress_file_t *comp_file;

148

149

comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;

150

dest_ctxt = comp_ctxt->dest_ctxt;

151

dest_ds = dest_ctxt->datasink;

152

153

/* Append the .qp extension to the filename */

154

fn_format(new_name, path, "", ".qp", MYF(MY_APPEND_EXT));

155

156

dest_file = dest_ds->open(dest_ctxt, new_name, mystat);

157

if (dest_file == NULL) {

158

return NULL;

159

}

160

161

/* Write the qpress archive header */

162

if (dest_ds->write(dest_file, "qpress10", 8) ||

163

write_uint64_le(dest_ds, dest_file, COMPRESS_CHUNK_SIZE)) {

164

goto err;

165

}

166

167

/* We are going to create a one-file "flat" (i.e. with no

168

subdirectories) archive. So strip the directory part from the path and

169

remove the '.qp' suffix. */

170

fn_format(new_name, path, "", "", MYF(MY_REPLACE_DIR));

171

172

/* Write the qpress file header */

173

name_len = strlen(new_name);

174

if (dest_ds->write(dest_file, "F", 1) ||

175

write_uint32_le(dest_ds, dest_file, name_len) ||

176

/* we want to write the terminating \0 as well */

177

dest_ds->write(dest_file, new_name, name_len + 1)) {

178

goto err;

179

}

180

181

file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +

182

sizeof(ds_compress_file_t),

183

MYF(MY_FAE));

184

comp_file = (ds_compress_file_t *) (file + 1);

185

comp_file->dest_file = dest_file;

186

comp_file->dest_ds = dest_ds;

187

comp_file->comp_ctxt = comp_ctxt;

188

comp_file->bytes_processed = 0;

189

190

file->ptr = comp_file;

191

file->path = dest_file->path;

192

193

return file;

194

195

err:

196

dest_ds->close(dest_file);

197

return NULL;

198

}

199

200

static

201

int

202

compress_write(ds_file_t *file, const void *buf, size_t len)

203

{

204

ds_compress_file_t *comp_file;

205

ds_compress_ctxt_t *comp_ctxt;

206

comp_thread_ctxt_t *threads;

207

comp_thread_ctxt_t *thd;

208

uint nthreads;

209

uint i;

210

const char *ptr;

211

datasink_t *dest_ds;

212

ds_file_t *dest_file;

213

214

comp_file = (ds_compress_file_t *) file->ptr;

215

comp_ctxt = comp_file->comp_ctxt;

216

dest_ds = comp_file->dest_ds;

217

dest_file = comp_file->dest_file;

218

219

threads = comp_ctxt->threads;

220

nthreads = comp_ctxt->nthreads;

221

222

ptr = (const char *) buf;

223

while (len > 0) {

224

uint max_thread;

225

226

/* Send data to worker threads for compression */

227

for (i = 0; i < nthreads; i++) {

228

size_t chunk_len;

229

230

thd = threads + i;

231

232

pthread_mutex_lock(&thd->ctrl_mutex);

233

234

chunk_len = (len > COMPRESS_CHUNK_SIZE) ?

235

COMPRESS_CHUNK_SIZE : len;

236

thd->from = ptr;

237

thd->from_len = chunk_len;

238

239

pthread_mutex_lock(&thd->data_mutex);

240

thd->data_avail = TRUE;

241

pthread_cond_signal(&thd->data_cond);

242

pthread_mutex_unlock(&thd->data_mutex);

243

244

len -= chunk_len;

245

if (len == 0) {

246

break;

247

}

248

ptr += chunk_len;

249

}

250

251

max_thread = (i < nthreads) ? i : nthreads - 1;

252

253

/* Reap and stream the compressed data */

254

for (i = 0; i <= max_thread; i++) {

255

thd = threads + i;

256

257

pthread_mutex_lock(&thd->data_mutex);

258

while (thd->data_avail == TRUE) {

259

pthread_cond_wait(&thd->data_cond,

260

&thd->data_mutex);

261

}

262

263

ut_a(threads[i].to_len > 0);

264

265

if (dest_ds->write(dest_file, "NEWBNEWB", 8) ||

266

write_uint64_le(dest_ds, dest_file,

267

comp_file->bytes_processed)) {

268

msg("compress: write to the destination stream "

269

"failed.\n");

270

return 1;

271

}

272

273

comp_file->bytes_processed += threads[i].from_len;

274

275

if (write_uint32_le(dest_ds, dest_file,

276

threads[i].adler) ||

277

dest_ds->write(dest_file, threads[i].to,

278

threads[i].to_len)) {

279

msg("compress: write to the destination stream "

280

"failed.\n");

281

return 1;

282

}

283

284

pthread_mutex_unlock(&threads[i].data_mutex);

285

pthread_mutex_unlock(&threads[i].ctrl_mutex);

286

}

287

}

288

289

return 0;

290

}

291

292

static

293

int

294

compress_close(ds_file_t *file)

295

{

296

ds_compress_file_t *comp_file;

297

datasink_t *dest_ds;

298

ds_file_t *dest_file;

299

300

comp_file = (ds_compress_file_t *) file->ptr;

301

dest_ds = comp_file->dest_ds;

302

dest_file = comp_file->dest_file;

303

304

/* Write the qpress file trailer */

305

dest_ds->write(dest_file, "ENDSENDS", 8);

306

307

/* Supposedly the number of written bytes should be written as a

308

"recovery information" in the file trailer, but in reality qpress

309

always writes 8 zeros here. Let's do the same */

310

311

write_uint64_le(dest_ds, dest_file, 0);

312

313

dest_ds->close(dest_file);

314

315

MY_FREE(file);

316

317

return 0;

318

}

319

320

static

321

void

322

compress_deinit(ds_ctxt_t *ctxt)

323

{

324

ds_compress_ctxt_t *comp_ctxt;

325

ds_ctxt_t *dest_ctxt;

326

datasink_t *dest_ds;

327

328

comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;;

329

330

destroy_worker_threads(comp_ctxt->threads, comp_ctxt->nthreads);

331

332

dest_ctxt = comp_ctxt->dest_ctxt;

333

dest_ds = dest_ctxt->datasink;

334

335

dest_ds->deinit(dest_ctxt);

336

337

MY_FREE(ctxt);

338

}

339

340

static inline

341

int

342

write_uint32_le(datasink_t *sink, ds_file_t *file, ulong n)

343

{

344

char tmp[4];

345

346

int4store(tmp, n);

347

return sink->write(file, tmp, sizeof(tmp));

348

}

349

350

static inline

351

int

352

write_uint64_le(datasink_t *sink, ds_file_t *file, ulonglong n)

353

{

354

char tmp[8];

355

356

int8store(tmp, n);

357

return sink->write(file, tmp, sizeof(tmp));

358

}

359

360

static

361

comp_thread_ctxt_t *

362

create_worker_threads(uint n)

363

{

364

comp_thread_ctxt_t *threads;

365

uint i;

366

367

threads = (comp_thread_ctxt_t *)

368

my_malloc(sizeof(comp_thread_ctxt_t) * n, MYF(MY_FAE));

369

370

for (i = 0; i < n; i++) {

371

comp_thread_ctxt_t *thd = threads + i;

372

373

thd->num = i + 1;

374

thd->started = FALSE;

375

thd->cancelled = FALSE;

376

thd->data_avail = FALSE;

377

378

thd->to = (char *) my_malloc(COMPRESS_CHUNK_SIZE +

379

MY_QLZ_COMPRESS_OVERHEAD,

380

MYF(MY_FAE));

381

382

/* Initialize the control mutex and condition var */

383

if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||

384

pthread_cond_init(&thd->ctrl_cond, NULL)) {

385

goto err;

386

}

387

388

/* Initialize and data mutex and condition var */

389

if (pthread_mutex_init(&thd->data_mutex, NULL) ||

390

pthread_cond_init(&thd->data_cond, NULL)) {

391

goto err;

392

}

393

394

pthread_mutex_lock(&thd->ctrl_mutex);

395

396

if (pthread_create(&thd->id, NULL, compress_worker_thread_func,

397

thd)) {

398

msg("compress: pthread_create() failed: "

399

"errno = %d\n", errno);

400

goto err;

401

}

402

}

403

404

/* Wait for the threads to start */

405

for (i = 0; i < n; i++) {

406

comp_thread_ctxt_t *thd = threads + i;

407

408

while (thd->started == FALSE)

409

pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);

410

pthread_mutex_unlock(&thd->ctrl_mutex);

411

}

412

413

return threads;

414

415

err:

416

return NULL;

417

}

418

419

static

420

void

421

destroy_worker_threads(comp_thread_ctxt_t *threads, uint n)

422

{

423

uint i;

424

425

for (i = 0; i < n; i++) {

426

comp_thread_ctxt_t *thd = threads + i;

427

428

pthread_mutex_lock(&thd->data_mutex);

429

threads[i].cancelled = TRUE;

430

pthread_cond_signal(&thd->data_cond);

431

pthread_mutex_unlock(&thd->data_mutex);

432

433

pthread_join(thd->id, NULL);

434

435

pthread_cond_destroy(&thd->data_cond);

436

pthread_mutex_destroy(&thd->data_mutex);

437

pthread_cond_destroy(&thd->ctrl_cond);

438

pthread_mutex_destroy(&thd->ctrl_mutex);

439

440

MY_FREE(thd->to);

441

}

442

443

MY_FREE(threads);

444

}

445

446

static

447

void *

448

compress_worker_thread_func(void *arg)

449

{

450

comp_thread_ctxt_t *thd = (comp_thread_ctxt_t *) arg;

451

452

pthread_mutex_lock(&thd->ctrl_mutex);

453

454

pthread_mutex_lock(&thd->data_mutex);

455

456

thd->started = TRUE;

457

pthread_cond_signal(&thd->ctrl_cond);

458

459

pthread_mutex_unlock(&thd->ctrl_mutex);

460

461

while (1) {

462

thd->data_avail = FALSE;

463

pthread_cond_signal(&thd->data_cond);

464

465

while (!thd->data_avail && !thd->cancelled) {

466

pthread_cond_wait(&thd->data_cond, &thd->data_mutex);

467

}

468

469

if (thd->cancelled)

470

break;

471

472

thd->to_len = qlz_compress(thd->from, thd->to, thd->from_len,

473

&thd->state);

474

475

/* qpress uses 0x00010000 as the initial value, but its own

476

Adler-32 implementation treats the value differently:

477

1. higher order bits are the sum of all bytes in the sequence

478

2. lower order bits are the sum of resulting values at every

479

step.

480

So it's the other way around as compared to zlib's adler32().

481

That's why 0x00000001 is being passed here to be compatible

482

with qpress implementation. */

483

484

thd->adler = adler32(0x00000001, (uchar *) thd->to,

485

thd->to_len);

486

}

487

488

pthread_mutex_unlock(&thd->data_mutex);

489

490

return NULL;

491

}

Older »