~gl-az/percona-xtrabackup/2.1-io-block-size

Viewing changes to src/libarchive/libarchive/archive_write.c

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: (391.1.5 staging)
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/libarchive/libarchive/archive_write.c

/*-

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR

* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "archive_platform.h"

__FBSDID("$FreeBSD: head/lib/libarchive/archive_write.c 201099 2009-12-28 03:03:00Z kientzle $");

* This file contains the "essential" portions of the write API, that

* is, stuff that will essentially always be used by any client that

* actually needs to write a archive. Optional pieces have been, as

* far as possible, separated out into separate files to reduce

* needlessly bloating statically-linked clients.

#ifdef HAVE_SYS_WAIT_H

#include <sys/wait.h>

#endif

#ifdef HAVE_LIMITS_H

#include <limits.h>

#endif

#include <stdio.h>

#ifdef HAVE_STDLIB_H

#include <stdlib.h>

#endif

#ifdef HAVE_STRING_H

#include <string.h>

#endif

#include <time.h>

#ifdef HAVE_UNISTD_H

#include <unistd.h>

#endif

#include "archive.h"

#include "archive_entry.h"

#include "archive_private.h"

#include "archive_write_private.h"

static struct archive_vtable *archive_write_vtable(void);

static int _archive_write_close(struct archive *);

static int _archive_write_finish(struct archive *);

static int _archive_write_header(struct archive *, struct archive_entry *);

static int _archive_write_finish_entry(struct archive *);

static ssize_t _archive_write_data(struct archive *, const void *, size_t);

static struct archive_vtable *

archive_write_vtable(void)

{

static struct archive_vtable av;

static int inited = 0;

if (!inited) {

av.archive_close = _archive_write_close;

av.archive_finish = _archive_write_finish;

av.archive_write_header = _archive_write_header;

av.archive_write_finish_entry = _archive_write_finish_entry;

av.archive_write_data = _archive_write_data;

}

return (&av);

}

* Allocate, initialize and return an archive object.

struct archive *

archive_write_new(void)

{

struct archive_write *a;

unsigned char *nulls;

a = (struct archive_write *)malloc(sizeof(*a));

if (a == NULL)

return (NULL);

memset(a, 0, sizeof(*a));

a->archive.magic = ARCHIVE_WRITE_MAGIC;

a->archive.state = ARCHIVE_STATE_NEW;

a->archive.vtable = archive_write_vtable();

100

101

* The value 10240 here matches the traditional tar default,

102

* but is otherwise arbitrary.

103

* TODO: Set the default block size from the format selected.

104

105

a->bytes_per_block = 10240;

106

a->bytes_in_last_block = -1; /* Default */

107

108

/* Initialize a block of nulls for padding purposes. */

109

a->null_length = 1024;

110

nulls = (unsigned char *)malloc(a->null_length);

111

if (nulls == NULL) {

112

free(a);

113

return (NULL);

114

}

115

memset(nulls, 0, a->null_length);

116

a->nulls = nulls;

117

118

* Set default compression, but don't set a default format.

119

* Were we to set a default format here, we would force every

120

* client to link in support for that format, even if they didn't

121

* ever use it.

122

123

archive_write_set_compression_none(&a->archive);

124

return (&a->archive);

125

}

126

127

128

* Set write options for the format. Returns 0 if successful.

129

130

int

131

archive_write_set_format_options(struct archive *_a, const char *s)

132

{

133

struct archive_write *a = (struct archive_write *)_a;

134

char key[64], val[64];

135

int len, r, ret = ARCHIVE_OK;

136

137

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

138

ARCHIVE_STATE_NEW, "archive_write_set_format_options");

139

archive_clear_error(&a->archive);

140

141

if (s == NULL || *s == '\0')

142

return (ARCHIVE_OK);

143

if (a->format_options == NULL)

144

/* This format does not support option. */

145

return (ARCHIVE_OK);

146

147

while ((len = __archive_parse_options(s, a->format_name,

148

sizeof(key), key, sizeof(val), val)) > 0) {

149

if (val[0] == '\0')

150

r = a->format_options(a, key, NULL);

151

else

152

r = a->format_options(a, key, val);

153

if (r == ARCHIVE_FATAL)

154

return (r);

155

if (r < ARCHIVE_OK) { /* This key was not handled. */

156

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

157

"Unsupported option ``%s''", key);

158

ret = ARCHIVE_WARN;

159

}

160

s += len;

161

}

162

if (len < 0) {

163

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

164

"Malformed options string.");

165

return (ARCHIVE_WARN);

166

}

167

return (ret);

168

}

169

170

171

* Set write options for the compressor. Returns 0 if successful.

172

173

int

174

archive_write_set_compressor_options(struct archive *_a, const char *s)

175

{

176

struct archive_write *a = (struct archive_write *)_a;

177

char key[64], val[64];

178

int len, r;

179

int ret = ARCHIVE_OK;

180

181

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

182

ARCHIVE_STATE_NEW, "archive_write_set_compressor_options");

183

archive_clear_error(&a->archive);

184

185

if (s == NULL || *s == '\0')

186

return (ARCHIVE_OK);

187

if (a->compressor.options == NULL) {

188

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

189

"Unsupported option ``%s''", s);

190

/* This compressor does not support option. */

191

return (ARCHIVE_WARN);

192

}

193

194

while ((len = __archive_parse_options(s, a->archive.compression_name,

195

sizeof(key), key, sizeof(val), val)) > 0) {

196

if (val[0] == '\0')

197

r = a->compressor.options(a, key, NULL);

198

else

199

r = a->compressor.options(a, key, val);

200

if (r == ARCHIVE_FATAL)

201

return (r);

202

if (r < ARCHIVE_OK) {

203

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

204

"Unsupported option ``%s''", key);

205

ret = ARCHIVE_WARN;

206

}

207

s += len;

208

}

209

if (len < 0) {

210

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

211

"Illegal format options.");

212

return (ARCHIVE_WARN);

213

}

214

return (ret);

215

}

216

217

218

* Set write options for the format and the compressor. Returns 0 if successful.

219

220

int

221

archive_write_set_options(struct archive *_a, const char *s)

222

{

223

int r1, r2;

224

225

r1 = archive_write_set_format_options(_a, s);

226

if (r1 < ARCHIVE_WARN)

227

return (r1);

228

r2 = archive_write_set_compressor_options(_a, s);

229

if (r2 < ARCHIVE_WARN)

230

return (r2);

231

if (r1 == ARCHIVE_WARN && r2 == ARCHIVE_WARN)

232

return (ARCHIVE_WARN);

233

return (ARCHIVE_OK);

234

}

235

236

237

* Set the block size. Returns 0 if successful.

238

239

int

240

archive_write_set_bytes_per_block(struct archive *_a, int bytes_per_block)

241

{

242

struct archive_write *a = (struct archive_write *)_a;

243

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

244

ARCHIVE_STATE_NEW, "archive_write_set_bytes_per_block");

245

a->bytes_per_block = bytes_per_block;

246

return (ARCHIVE_OK);

247

}

248

249

250

* Get the current block size. -1 if it has never been set.

251

252

int

253

archive_write_get_bytes_per_block(struct archive *_a)

254

{

255

struct archive_write *a = (struct archive_write *)_a;

256

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

257

ARCHIVE_STATE_ANY, "archive_write_get_bytes_per_block");

258

return (a->bytes_per_block);

259

}

260

261

262

* Set the size for the last block.

263

* Returns 0 if successful.

264

265

int

266

archive_write_set_bytes_in_last_block(struct archive *_a, int bytes)

267

{

268

struct archive_write *a = (struct archive_write *)_a;

269

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

270

ARCHIVE_STATE_ANY, "archive_write_set_bytes_in_last_block");

271

a->bytes_in_last_block = bytes;

272

return (ARCHIVE_OK);

273

}

274

275

276

* Return the value set above. -1 indicates it has not been set.

277

278

int

279

archive_write_get_bytes_in_last_block(struct archive *_a)

280

{

281

struct archive_write *a = (struct archive_write *)_a;

282

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

283

ARCHIVE_STATE_ANY, "archive_write_get_bytes_in_last_block");

284

return (a->bytes_in_last_block);

285

}

286

287

288

289

* dev/ino of a file to be rejected. Used to prevent adding

290

* an archive to itself recursively.

291

292

int

293

archive_write_set_skip_file(struct archive *_a, dev_t d, ino_t i)

294

{

295

struct archive_write *a = (struct archive_write *)_a;

296

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

297

ARCHIVE_STATE_ANY, "archive_write_set_skip_file");

298

a->skip_file_dev = d;

299

a->skip_file_ino = i;

300

return (ARCHIVE_OK);

301

}

302

303

304

305

* Open the archive using the current settings.

306

307

int

308

archive_write_open(struct archive *_a, void *client_data,

309

archive_open_callback *opener, archive_write_callback *writer,

310

archive_close_callback *closer)

311

{

312

struct archive_write *a = (struct archive_write *)_a;

313

int ret;

314

315

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

316

ARCHIVE_STATE_NEW, "archive_write_open");

317

archive_clear_error(&a->archive);

318

a->archive.state = ARCHIVE_STATE_HEADER;

319

a->client_data = client_data;

320

a->client_writer = writer;

321

a->client_opener = opener;

322

a->client_closer = closer;

323

ret = (a->compressor.init)(a);

324

if (a->format_init && ret == ARCHIVE_OK)

325

ret = (a->format_init)(a);

326

return (ret);

327

}

328

329

330

331

* Close out the archive.

332

333

* Be careful: user might just call write_new and then write_finish.

334

* Don't assume we actually wrote anything or performed any non-trivial

335

* initialization.

336

337

static int

338

_archive_write_close(struct archive *_a)

339

{

340

struct archive_write *a = (struct archive_write *)_a;

341

int r = ARCHIVE_OK, r1 = ARCHIVE_OK;

342

343

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

344

ARCHIVE_STATE_ANY, "archive_write_close");

345

346

/* Finish the last entry. */

347

if (a->archive.state & ARCHIVE_STATE_DATA)

348

r = ((a->format_finish_entry)(a));

349

350

/* Finish off the archive. */

351

if (a->format_finish != NULL) {

352

r1 = (a->format_finish)(a);

353

if (r1 < r)

354

r = r1;

355

}

356

357

/* Release format resources. */

358

if (a->format_destroy != NULL) {

359

r1 = (a->format_destroy)(a);

360

if (r1 < r)

361

r = r1;

362

}

363

364

/* Finish the compression and close the stream. */

365

if (a->compressor.finish != NULL) {

366

r1 = (a->compressor.finish)(a);

367

if (r1 < r)

368

r = r1;

369

}

370

371

/* Close out the client stream. */

372

if (a->client_closer != NULL) {

373

r1 = (a->client_closer)(&a->archive, a->client_data);

374

if (r1 < r)

375

r = r1;

376

}

377

378

a->archive.state = ARCHIVE_STATE_CLOSED;

379

return (r);

380

}

381

382

383

* Destroy the archive structure.

384

385

static int

386

_archive_write_finish(struct archive *_a)

387

{

388

struct archive_write *a = (struct archive_write *)_a;

389

int r = ARCHIVE_OK;

390

391

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

392

ARCHIVE_STATE_ANY, "archive_write_finish");

393

if (a->archive.state != ARCHIVE_STATE_CLOSED)

394

r = archive_write_close(&a->archive);

395

396

/* Release various dynamic buffers. */

397

free((void *)(uintptr_t)(const void *)a->nulls);

398

archive_string_free(&a->archive.error_string);

399

a->archive.magic = 0;

400

free(a);

401

return (r);

402

}

403

404

405

* Write the appropriate header.

406

407

static int

408

_archive_write_header(struct archive *_a, struct archive_entry *entry)

409

{

410

struct archive_write *a = (struct archive_write *)_a;

411

int ret, r2;

412

413

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

414

ARCHIVE_STATE_DATA | ARCHIVE_STATE_HEADER, "archive_write_header");

415

archive_clear_error(&a->archive);

416

417

/* In particular, "retry" and "fatal" get returned immediately. */

418

ret = archive_write_finish_entry(&a->archive);

419

if (ret < ARCHIVE_OK && ret != ARCHIVE_WARN)

420

return (ret);

421

422

if (a->skip_file_dev != 0 &&

423

archive_entry_dev(entry) == a->skip_file_dev &&

424

a->skip_file_ino != 0 &&

425

archive_entry_ino64(entry) == a->skip_file_ino) {

426

archive_set_error(&a->archive, 0,

427

"Can't add archive to itself");

428

return (ARCHIVE_FAILED);

429

}

430

431

/* Format and write header. */

432

r2 = ((a->format_write_header)(a, entry));

433

if (r2 < ret)

434

ret = r2;

435

436

a->archive.state = ARCHIVE_STATE_DATA;

437

return (ret);

438

}

439

440

static int

441

_archive_write_finish_entry(struct archive *_a)

442

{

443

struct archive_write *a = (struct archive_write *)_a;

444

int ret = ARCHIVE_OK;

445

446

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

447

ARCHIVE_STATE_HEADER | ARCHIVE_STATE_DATA,

448

"archive_write_finish_entry");

449

if (a->archive.state & ARCHIVE_STATE_DATA)

450

ret = (a->format_finish_entry)(a);

451

a->archive.state = ARCHIVE_STATE_HEADER;

452

return (ret);

453

}

454

455

456

* Note that the compressor is responsible for blocking.

457

458

static ssize_t

459

_archive_write_data(struct archive *_a, const void *buff, size_t s)

460

{

461

struct archive_write *a = (struct archive_write *)_a;

462

__archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,

463

ARCHIVE_STATE_DATA, "archive_write_data");

464

archive_clear_error(&a->archive);

465

return ((a->format_write_data)(a, buff, s));

466

}

Older »