~akopytov/percona-xtrabackup/bug1166888-2.1

Viewing changes to src/libarchive/libarchive/archive_write_set_format_ar.c

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: (391.1.5 staging)
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/libarchive/libarchive/archive_write_set_format_ar.c

/*-

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer

* in this position and unchanged.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR

* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "archive_platform.h"

__FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_format_ar.c 201108 2009-12-28 03:28:21Z kientzle $");

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#ifdef HAVE_STDLIB_H

#include <stdlib.h>

#endif

#ifdef HAVE_STRING_H

#include <string.h>

#endif

#include "archive.h"

#include "archive_entry.h"

#include "archive_private.h"

#include "archive_write_private.h"

struct ar_w {

uint64_t entry_bytes_remaining;

uint64_t entry_padding;

int is_strtab;

int has_strtab;

char *strtab;

};

* Define structure of the "ar" header.

#define AR_name_offset 0

#define AR_name_size 16

#define AR_date_offset 16

#define AR_date_size 12

#define AR_uid_offset 28

#define AR_uid_size 6

#define AR_gid_offset 34

#define AR_gid_size 6

#define AR_mode_offset 40

#define AR_mode_size 8

#define AR_size_offset 48

#define AR_size_size 10

#define AR_fmag_offset 58

#define AR_fmag_size 2

static int archive_write_set_format_ar(struct archive_write *);

static int archive_write_ar_header(struct archive_write *,

struct archive_entry *);

static ssize_t archive_write_ar_data(struct archive_write *,

const void *buff, size_t s);

static int archive_write_ar_destroy(struct archive_write *);

static int archive_write_ar_finish(struct archive_write *);

static int archive_write_ar_finish_entry(struct archive_write *);

static const char *ar_basename(const char *path);

static int format_octal(int64_t v, char *p, int s);

static int format_decimal(int64_t v, char *p, int s);

int

archive_write_set_format_ar_bsd(struct archive *_a)

{

struct archive_write *a = (struct archive_write *)_a;

int r = archive_write_set_format_ar(a);

if (r == ARCHIVE_OK) {

a->archive.archive_format = ARCHIVE_FORMAT_AR_BSD;

a->archive.archive_format_name = "ar (BSD)";

}

return (r);

}

int

archive_write_set_format_ar_svr4(struct archive *_a)

{

struct archive_write *a = (struct archive_write *)_a;

100

int r = archive_write_set_format_ar(a);

101

if (r == ARCHIVE_OK) {

102

a->archive.archive_format = ARCHIVE_FORMAT_AR_GNU;

103

a->archive.archive_format_name = "ar (GNU/SVR4)";

104

}

105

return (r);

106

}

107

108

109

* Generic initialization.

110

111

static int

112

archive_write_set_format_ar(struct archive_write *a)

113

{

114

struct ar_w *ar;

115

116

/* If someone else was already registered, unregister them. */

117

if (a->format_destroy != NULL)

118

(a->format_destroy)(a);

119

120

ar = (struct ar_w *)malloc(sizeof(*ar));

121

if (ar == NULL) {

122

archive_set_error(&a->archive, ENOMEM, "Can't allocate ar data");

123

return (ARCHIVE_FATAL);

124

}

125

memset(ar, 0, sizeof(*ar));

126

a->format_data = ar;

127

128

a->format_name = "ar";

129

a->format_write_header = archive_write_ar_header;

130

a->format_write_data = archive_write_ar_data;

131

a->format_finish = archive_write_ar_finish;

132

a->format_destroy = archive_write_ar_destroy;

133

a->format_finish_entry = archive_write_ar_finish_entry;

134

return (ARCHIVE_OK);

135

}

136

137

static int

138

archive_write_ar_header(struct archive_write *a, struct archive_entry *entry)

139

{

140

int ret, append_fn;

141

char buff[60];

142

char *ss, *se;

143

struct ar_w *ar;

144

const char *pathname;

145

const char *filename;

146

int64_t size;

147

148

append_fn = 0;

149

ar = (struct ar_w *)a->format_data;

150

ar->is_strtab = 0;

151

filename = NULL;

152

size = archive_entry_size(entry);

153

154

155

156

* Reject files with empty name.

157

158

pathname = archive_entry_pathname(entry);

159

if (*pathname == '\0') {

160

archive_set_error(&a->archive, EINVAL,

161

"Invalid filename");

162

return (ARCHIVE_WARN);

163

}

164

165

166

* If we are now at the beginning of the archive,

167

* we need first write the ar global header.

168

169

if (a->archive.file_position == 0)

170

(a->compressor.write)(a, "!<arch>\n", 8);

171

172

memset(buff, ' ', 60);

173

strncpy(&buff[AR_fmag_offset], "`\n", 2);

174

175

if (strcmp(pathname, "/") == 0 ) {

176

/* Entry is archive symbol table in GNU format */

177

buff[AR_name_offset] = '/';

178

goto stat;

179

}

180

if (strcmp(pathname, "__.SYMDEF") == 0) {

181

/* Entry is archive symbol table in BSD format */

182

strncpy(buff + AR_name_offset, "__.SYMDEF", 9);

183

goto stat;

184

}

185

if (strcmp(pathname, "//") == 0) {

186

187

* Entry is archive filename table, inform that we should

188

* collect strtab in next _data call.

189

190

ar->is_strtab = 1;

191

buff[AR_name_offset] = buff[AR_name_offset + 1] = '/';

192

193

* For archive string table, only ar_size filed should

194

* be set.

195

196

goto size;

197

}

198

199

200

* Otherwise, entry is a normal archive member.

201

* Strip leading paths from filenames, if any.

202

203

if ((filename = ar_basename(pathname)) == NULL) {

204

/* Reject filenames with trailing "/" */

205

archive_set_error(&a->archive, EINVAL,

206

"Invalid filename");

207

return (ARCHIVE_WARN);

208

}

209

210

if (a->archive.archive_format == ARCHIVE_FORMAT_AR_GNU) {

211

212

* SVR4/GNU variant use a "/" to mark then end of the filename,

213

* make it possible to have embedded spaces in the filename.

214

* So, the longest filename here (without extension) is

215

* actually 15 bytes.

216

217

if (strlen(filename) <= 15) {

218

strncpy(&buff[AR_name_offset],

219

filename, strlen(filename));

220

buff[AR_name_offset + strlen(filename)] = '/';

221

} else {

222

223

* For filename longer than 15 bytes, GNU variant

224

* makes use of a string table and instead stores the

225

* offset of the real filename to in the ar_name field.

226

* The string table should have been written before.

227

228

if (ar->has_strtab <= 0) {

229

archive_set_error(&a->archive, EINVAL,

230

"Can't find string table");

231

return (ARCHIVE_WARN);

232

}

233

234

se = (char *)malloc(strlen(filename) + 3);

235

if (se == NULL) {

236

archive_set_error(&a->archive, ENOMEM,

237

"Can't allocate filename buffer");

238

return (ARCHIVE_FATAL);

239

}

240

241

strncpy(se, filename, strlen(filename));

242

strcpy(se + strlen(filename), "/\n");

243

244

ss = strstr(ar->strtab, se);

245

free(se);

246

247

if (ss == NULL) {

248

archive_set_error(&a->archive, EINVAL,

249

"Invalid string table");

250

return (ARCHIVE_WARN);

251

}

252

253

254

* GNU variant puts "/" followed by digits into

255

* ar_name field. These digits indicates the real

256

* filename string's offset to the string table.

257

258

buff[AR_name_offset] = '/';

259

if (format_decimal(ss - ar->strtab,

260

buff + AR_name_offset + 1,

261

AR_name_size - 1)) {

262

archive_set_error(&a->archive, ERANGE,

263

"string table offset too large");

264

return (ARCHIVE_WARN);

265

}

266

}

267

} else if (a->archive.archive_format == ARCHIVE_FORMAT_AR_BSD) {

268

269

* BSD variant: for any file name which is more than

270

* 16 chars or contains one or more embedded space(s), the

271

* string "#1/" followed by the ASCII length of the name is

272

* put into the ar_name field. The file size (stored in the

273

* ar_size field) is incremented by the length of the name.

274

* The name is then written immediately following the

275

* archive header.

276

277

if (strlen(filename) <= 16 && strchr(filename, ' ') == NULL) {

278

strncpy(&buff[AR_name_offset], filename, strlen(filename));

279

buff[AR_name_offset + strlen(filename)] = ' ';

280

}

281

else {

282

strncpy(buff + AR_name_offset, "#1/", 3);

283

if (format_decimal(strlen(filename),

284

buff + AR_name_offset + 3,

285

AR_name_size - 3)) {

286

archive_set_error(&a->archive, ERANGE,

287

"File name too long");

288

return (ARCHIVE_WARN);

289

}

290

append_fn = 1;

291

size += strlen(filename);

292

}

293

}

294

295

stat:

296

if (format_decimal(archive_entry_mtime(entry), buff + AR_date_offset, AR_date_size)) {

297

archive_set_error(&a->archive, ERANGE,

298

"File modification time too large");

299

return (ARCHIVE_WARN);

300

}

301

if (format_decimal(archive_entry_uid(entry), buff + AR_uid_offset, AR_uid_size)) {

302

archive_set_error(&a->archive, ERANGE,

303

"Numeric user ID too large");

304

return (ARCHIVE_WARN);

305

}

306

if (format_decimal(archive_entry_gid(entry), buff + AR_gid_offset, AR_gid_size)) {

307

archive_set_error(&a->archive, ERANGE,

308

"Numeric group ID too large");

309

return (ARCHIVE_WARN);

310

}

311

if (format_octal(archive_entry_mode(entry), buff + AR_mode_offset, AR_mode_size)) {

312

archive_set_error(&a->archive, ERANGE,

313

"Numeric mode too large");

314

return (ARCHIVE_WARN);

315

}

316

317

* Sanity Check: A non-pseudo archive member should always be

318

* a regular file.

319

320

if (filename != NULL && archive_entry_filetype(entry) != AE_IFREG) {

321

archive_set_error(&a->archive, EINVAL,

322

"Regular file required for non-pseudo member");

323

return (ARCHIVE_WARN);

324

}

325

326

size:

327

if (format_decimal(size, buff + AR_size_offset, AR_size_size)) {

328

archive_set_error(&a->archive, ERANGE,

329

"File size out of range");

330

return (ARCHIVE_WARN);

331

}

332

333

ret = (a->compressor.write)(a, buff, 60);

334

if (ret != ARCHIVE_OK)

335

return (ret);

336

337

ar->entry_bytes_remaining = size;

338

ar->entry_padding = ar->entry_bytes_remaining % 2;

339

340

if (append_fn > 0) {

341

ret = (a->compressor.write)(a, filename, strlen(filename));

342

if (ret != ARCHIVE_OK)

343

return (ret);

344

ar->entry_bytes_remaining -= strlen(filename);

345

}

346

347

return (ARCHIVE_OK);

348

}

349

350

static ssize_t

351

archive_write_ar_data(struct archive_write *a, const void *buff, size_t s)

352

{

353

struct ar_w *ar;

354

int ret;

355

356

ar = (struct ar_w *)a->format_data;

357

if (s > ar->entry_bytes_remaining)

358

s = ar->entry_bytes_remaining;

359

360

if (ar->is_strtab > 0) {

361

if (ar->has_strtab > 0) {

362

archive_set_error(&a->archive, EINVAL,

363

"More than one string tables exist");

364

return (ARCHIVE_WARN);

365

}

366

367

ar->strtab = (char *)malloc(s);

368

if (ar->strtab == NULL) {

369

archive_set_error(&a->archive, ENOMEM,

370

"Can't allocate strtab buffer");

371

return (ARCHIVE_FATAL);

372

}

373

strncpy(ar->strtab, buff, s);

374

ar->has_strtab = 1;

375

}

376

377

ret = (a->compressor.write)(a, buff, s);

378

if (ret != ARCHIVE_OK)

379

return (ret);

380

381

ar->entry_bytes_remaining -= s;

382

return (s);

383

}

384

385

static int

386

archive_write_ar_destroy(struct archive_write *a)

387

{

388

struct ar_w *ar;

389

390

ar = (struct ar_w *)a->format_data;

391

392

if (ar == NULL)

393

return (ARCHIVE_OK);

394

395

if (ar->has_strtab > 0) {

396

free(ar->strtab);

397

ar->strtab = NULL;

398

}

399

400

free(ar);

401

a->format_data = NULL;

402

return (ARCHIVE_OK);

403

}

404

405

static int

406

archive_write_ar_finish(struct archive_write *a)

407

{

408

int ret;

409

410

411

* If we haven't written anything yet, we need to write

412

* the ar global header now to make it a valid ar archive.

413

414

if (a->archive.file_position == 0) {

415

ret = (a->compressor.write)(a, "!<arch>\n", 8);

416

return (ret);

417

}

418

419

return (ARCHIVE_OK);

420

}

421

422

static int

423

archive_write_ar_finish_entry(struct archive_write *a)

424

{

425

struct ar_w *ar;

426

int ret;

427

428

ar = (struct ar_w *)a->format_data;

429

430

if (ar->entry_bytes_remaining != 0) {

431

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

432

"Entry remaining bytes larger than 0");

433

return (ARCHIVE_WARN);

434

}

435

436

if (ar->entry_padding == 0) {

437

return (ARCHIVE_OK);

438

}

439

440

if (ar->entry_padding != 1) {

441

archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,

442

"Padding wrong size: %d should be 1 or 0",

443

ar->entry_padding);

444

return (ARCHIVE_WARN);

445

}

446

447

ret = (a->compressor.write)(a, "\n", 1);

448

return (ret);

449

}

450

451

452

* Format a number into the specified field using base-8.

453

* NB: This version is slightly different from the one in

454

* _ustar.c

455

456

static int

457

format_octal(int64_t v, char *p, int s)

458

{

459

int len;

460

char *h;

461

462

len = s;

463

h = p;

464

465

/* Octal values can't be negative, so use 0. */

466

if (v < 0) {

467

while (len-- > 0)

468

*p++ = '0';

469

return (-1);

470

}

471

472

p += s; /* Start at the end and work backwards. */

473

do {

474

*--p = (char)('0' + (v & 7));

475

v >>= 3;

476

} while (--s > 0 && v > 0);

477

478

if (v == 0) {

479

memmove(h, p, len - s);

480

p = h + len - s;

481

while (s-- > 0)

482

*p++ = ' ';

483

return (0);

484

}

485

/* If it overflowed, fill field with max value. */

486

while (len-- > 0)

487

*p++ = '7';

488

489

return (-1);

490

}

491

492

493

* Format a number into the specified field using base-10.

494

495

static int

496

format_decimal(int64_t v, char *p, int s)

497

{

498

int len;

499

char *h;

500

501

len = s;

502

h = p;

503

504

/* Negative values in ar header are meaningless , so use 0. */

505

if (v < 0) {

506

while (len-- > 0)

507

*p++ = '0';

508

return (-1);

509

}

510

511

p += s;

512

do {

513

*--p = (char)('0' + (v % 10));

514

v /= 10;

515

} while (--s > 0 && v > 0);

516

517

if (v == 0) {

518

memmove(h, p, len - s);

519

p = h + len - s;

520

while (s-- > 0)

521

*p++ = ' ';

522

return (0);

523

}

524

/* If it overflowed, fill field with max value. */

525

while (len-- > 0)

526

*p++ = '9';

527

528

return (-1);

529

}

530

531

static const char *

532

ar_basename(const char *path)

533

{

534

const char *endp, *startp;

535

536

endp = path + strlen(path) - 1;

537

538

* For filename with trailing slash(es), we return

539

* NULL indicating an error.

540

541

if (*endp == '/')

542

return (NULL);

543

544

/* Find the start of the base */

545

startp = endp;

546

while (startp > path && *(startp - 1) != '/')

547

startp--;

548

549

return (startp);

550

}

Older »