~akopytov/percona-xtrabackup/bug1166888-2.1

Viewing changes to src/libarchive/tar/read.c

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: (391.1.5 staging)
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/libarchive/tar/read.c

/*-

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR

* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "bsdtar_platform.h"

__FBSDID("$FreeBSD: src/usr.bin/tar/read.c,v 1.40 2008/08/21 06:41:14 kientzle Exp $");

#ifdef HAVE_SYS_TYPES_H

#include <sys/types.h>

#endif

#ifdef HAVE_SYS_PARAM_H

#include <sys/param.h>

#endif

#ifdef HAVE_SYS_STAT_H

#include <sys/stat.h>

#endif

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#ifdef HAVE_GRP_H

#include <grp.h>

#endif

#ifdef HAVE_LIMITS_H

#include <limits.h>

#endif

#ifdef HAVE_PWD_H

#include <pwd.h>

#endif

#ifdef HAVE_STDINT_H

#include <stdint.h>

#endif

#include <stdio.h>

#ifdef HAVE_STDLIB_H

#include <stdlib.h>

#endif

#ifdef HAVE_STRING_H

#include <string.h>

#endif

#ifdef HAVE_TIME_H

#include <time.h>

#endif

#ifdef HAVE_UNISTD_H

#include <unistd.h>

#endif

#include "bsdtar.h"

#include "err.h"

struct progress_data {

struct bsdtar *bsdtar;

struct archive *archive;

struct archive_entry *entry;

};

static void list_item_verbose(struct bsdtar *, FILE *,

struct archive_entry *);

static void read_archive(struct bsdtar *bsdtar, char mode);

void

tar_mode_t(struct bsdtar *bsdtar)

{

read_archive(bsdtar, 't');

if (lafe_unmatched_inclusions_warn(bsdtar->matching, "Not found in archive") != 0)

bsdtar->return_value = 1;

}

void

tar_mode_x(struct bsdtar *bsdtar)

{

read_archive(bsdtar, 'x');

if (lafe_unmatched_inclusions_warn(bsdtar->matching, "Not found in archive") != 0)

bsdtar->return_value = 1;

}

static void

progress_func(void *cookie)

100

{

101

struct progress_data *progress_data = cookie;

102

struct bsdtar *bsdtar = progress_data->bsdtar;

103

struct archive *a = progress_data->archive;

104

struct archive_entry *entry = progress_data->entry;

105

uint64_t comp, uncomp;

106

107

if (!need_report())

108

return;

109

110

if (bsdtar->verbose)

111

fprintf(stderr, "\n");

112

if (a != NULL) {

113

comp = archive_position_compressed(a);

114

uncomp = archive_position_uncompressed(a);

115

fprintf(stderr,

116

"In: %s bytes, compression %d%%;",

117

tar_i64toa(comp), (int)((uncomp - comp) * 100 / uncomp));

118

fprintf(stderr, " Out: %d files, %s bytes\n",

119

archive_file_count(a), tar_i64toa(uncomp));

120

}

121

if (entry != NULL) {

122

safe_fprintf(stderr, "Current: %s",

123

archive_entry_pathname(entry));

124

fprintf(stderr, " (%s bytes)\n",

125

tar_i64toa(archive_entry_size(entry)));

126

}

127

}

128

129

130

* Handle 'x' and 't' modes.

131

132

static void

133

read_archive(struct bsdtar *bsdtar, char mode)

134

{

135

struct progress_data progress_data;

136

FILE *out;

137

struct archive *a;

138

struct archive_entry *entry;

139

const struct stat *st;

140

int r;

141

142

while (*bsdtar->argv) {

143

lafe_include(&bsdtar->matching, *bsdtar->argv);

144

bsdtar->argv++;

145

}

146

147

if (bsdtar->names_from_file != NULL)

148

lafe_include_from_file(&bsdtar->matching,

149

bsdtar->names_from_file, bsdtar->option_null);

150

151

a = archive_read_new();

152

if (bsdtar->compress_program != NULL)

153

archive_read_support_compression_program(a, bsdtar->compress_program);

154

else

155

archive_read_support_compression_all(a);

156

archive_read_support_format_all(a);

157

if (ARCHIVE_OK != archive_read_set_options(a, bsdtar->option_options))

158

lafe_errc(1, 0, "%s", archive_error_string(a));

159

if (archive_read_open_file(a, bsdtar->filename,

160

bsdtar->bytes_per_block != 0 ? bsdtar->bytes_per_block :

161

DEFAULT_BYTES_PER_BLOCK))

162

lafe_errc(1, 0, "Error opening archive: %s",

163

archive_error_string(a));

164

165

do_chdir(bsdtar);

166

167

if (mode == 'x') {

168

/* Set an extract callback so that we can handle SIGINFO. */

169

progress_data.bsdtar = bsdtar;

170

progress_data.archive = a;

171

archive_read_extract_set_progress_callback(a, progress_func,

172

&progress_data);

173

}

174

175

if (mode == 'x' && bsdtar->option_chroot) {

176

#if HAVE_CHROOT

177

if (chroot(".") != 0)

178

lafe_errc(1, errno, "Can't chroot to \".\"");

179

#else

180

lafe_errc(1, 0,

181

"chroot isn't supported on this platform");

182

#endif

183

}

184

185

for (;;) {

186

/* Support --fast-read option */

187

if (bsdtar->option_fast_read &&

188

lafe_unmatched_inclusions(bsdtar->matching) == 0)

189

break;

190

191

r = archive_read_next_header(a, &entry);

192

progress_data.entry = entry;

193

if (r == ARCHIVE_EOF)

194

break;

195

if (r < ARCHIVE_OK)

196

lafe_warnc(0, "%s", archive_error_string(a));

197

if (r <= ARCHIVE_WARN)

198

bsdtar->return_value = 1;

199

if (r == ARCHIVE_RETRY) {

200

/* Retryable error: try again */

201

lafe_warnc(0, "Retrying...");

202

continue;

203

}

204

if (r == ARCHIVE_FATAL)

205

break;

206

207

if (bsdtar->option_numeric_owner) {

208

archive_entry_set_uname(entry, NULL);

209

archive_entry_set_gname(entry, NULL);

210

}

211

212

213

* Exclude entries that are too old.

214

215

st = archive_entry_stat(entry);

216

if (bsdtar->newer_ctime_sec > 0) {

217

if (st->st_ctime < bsdtar->newer_ctime_sec)

218

continue; /* Too old, skip it. */

219

if (st->st_ctime == bsdtar->newer_ctime_sec

220

&& ARCHIVE_STAT_CTIME_NANOS(st)

221

<= bsdtar->newer_ctime_nsec)

222

continue; /* Too old, skip it. */

223

}

224

if (bsdtar->newer_mtime_sec > 0) {

225

if (st->st_mtime < bsdtar->newer_mtime_sec)

226

continue; /* Too old, skip it. */

227

if (st->st_mtime == bsdtar->newer_mtime_sec

228

&& ARCHIVE_STAT_MTIME_NANOS(st)

229

<= bsdtar->newer_mtime_nsec)

230

continue; /* Too old, skip it. */

231

}

232

233

234

* Note that pattern exclusions are checked before

235

* pathname rewrites are handled. This gives more

236

* control over exclusions, since rewrites always lose

237

* information. (For example, consider a rewrite

238

* s/foo[0-9]/foo/. If we check exclusions after the

239

* rewrite, there would be no way to exclude foo1/bar

240

* while allowing foo2/bar.)

241

242

if (lafe_excluded(bsdtar->matching, archive_entry_pathname(entry)))

243

continue; /* Excluded by a pattern test. */

244

245

if (mode == 't') {

246

/* Perversely, gtar uses -O to mean "send to stderr"

247

* when used with -t. */

248

out = bsdtar->option_stdout ? stderr : stdout;

249

250

251

* TODO: Provide some reasonable way to

252

* preview rewrites. gtar always displays

253

* the unedited path in -t output, which means

254

* you cannot easily preview rewrites.

255

256

if (bsdtar->verbose < 2)

257

safe_fprintf(out, "%s",

258

archive_entry_pathname(entry));

259

else

260

list_item_verbose(bsdtar, out, entry);

261

fflush(out);

262

r = archive_read_data_skip(a);

263

if (r == ARCHIVE_WARN) {

264

fprintf(out, "\n");

265

lafe_warnc(0, "%s",

266

archive_error_string(a));

267

}

268

if (r == ARCHIVE_RETRY) {

269

fprintf(out, "\n");

270

lafe_warnc(0, "%s",

271

archive_error_string(a));

272

}

273

if (r == ARCHIVE_FATAL) {

274

fprintf(out, "\n");

275

lafe_warnc(0, "%s",

276

archive_error_string(a));

277

bsdtar->return_value = 1;

278

break;

279

}

280

fprintf(out, "\n");

281

} else {

282

/* Note: some rewrite failures prevent extraction. */

283

if (edit_pathname(bsdtar, entry))

284

continue; /* Excluded by a rewrite failure. */

285

286

if (bsdtar->option_interactive &&

287

!yes("extract '%s'", archive_entry_pathname(entry)))

288

continue;

289

290

291

* Format here is from SUSv2, including the

292

* deferred '\n'.

293

294

if (bsdtar->verbose) {

295

safe_fprintf(stderr, "x %s",

296

archive_entry_pathname(entry));

297

fflush(stderr);

298

}

299

300

// TODO siginfo_printinfo(bsdtar, 0);

301

302

if (bsdtar->option_stdout)

303

r = archive_read_data_into_fd(a, 1);

304

else

305

r = archive_read_extract(a, entry,

306

bsdtar->extract_flags);

307

if (r != ARCHIVE_OK) {

308

if (!bsdtar->verbose)

309

safe_fprintf(stderr, "%s",

310

archive_entry_pathname(entry));

311

safe_fprintf(stderr, ": %s",

312

archive_error_string(a));

313

if (!bsdtar->verbose)

314

fprintf(stderr, "\n");

315

bsdtar->return_value = 1;

316

}

317

if (bsdtar->verbose)

318

fprintf(stderr, "\n");

319

if (r == ARCHIVE_FATAL)

320

break;

321

}

322

}

323

324

325

r = archive_read_close(a);

326

if (r != ARCHIVE_OK)

327

lafe_warnc(0, "%s", archive_error_string(a));

328

if (r <= ARCHIVE_WARN)

329

bsdtar->return_value = 1;

330

331

if (bsdtar->verbose > 2)

332

fprintf(stdout, "Archive Format: %s, Compression: %s\n",

333

archive_format_name(a), archive_compression_name(a));

334

335

archive_read_finish(a);

336

}

337

338

339

340

* Display information about the current file.

341

342

* The format here roughly duplicates the output of 'ls -l'.

343

* This is based on SUSv2, where 'tar tv' is documented as

344

* listing additional information in an "unspecified format,"

345

* and 'pax -l' is documented as using the same format as 'ls -l'.

346

347

static void

348

list_item_verbose(struct bsdtar *bsdtar, FILE *out, struct archive_entry *entry)

349

{

350

char tmp[100];

351

size_t w;

352

const char *p;

353

const char *fmt;

354

time_t tim;

355

static time_t now;

356

357

358

* We avoid collecting the entire list in memory at once by

359

* listing things as we see them. However, that also means we can't

360

* just pre-compute the field widths. Instead, we start with guesses

361

* and just widen them as necessary. These numbers are completely

362

* arbitrary.

363

364

if (!bsdtar->u_width) {

365

bsdtar->u_width = 6;

366

bsdtar->gs_width = 13;

367

}

368

if (!now)

369

time(&now);

370

fprintf(out, "%s %d ",

371

archive_entry_strmode(entry),

372

archive_entry_nlink(entry));

373

374

/* Use uname if it's present, else uid. */

375

p = archive_entry_uname(entry);

376

if ((p == NULL) || (*p == '\0')) {

377

sprintf(tmp, "%lu ",

378

(unsigned long)archive_entry_uid(entry));

379

p = tmp;

380

}

381

w = strlen(p);

382

if (w > bsdtar->u_width)

383

bsdtar->u_width = w;

384

fprintf(out, "%-*s ", (int)bsdtar->u_width, p);

385

386

/* Use gname if it's present, else gid. */

387

p = archive_entry_gname(entry);

388

if (p != NULL && p[0] != '\0') {

389

fprintf(out, "%s", p);

390

w = strlen(p);

391

} else {

392

sprintf(tmp, "%lu",

393

(unsigned long)archive_entry_gid(entry));

394

w = strlen(tmp);

395

fprintf(out, "%s", tmp);

396

}

397

398

399

* Print device number or file size, right-aligned so as to make

400

* total width of group and devnum/filesize fields be gs_width.

401

* If gs_width is too small, grow it.

402

403

if (archive_entry_filetype(entry) == AE_IFCHR

404

|| archive_entry_filetype(entry) == AE_IFBLK) {

405

sprintf(tmp, "%lu,%lu",

406

(unsigned long)archive_entry_rdevmajor(entry),

407

(unsigned long)archive_entry_rdevminor(entry));

408

} else {

409

strcpy(tmp, tar_i64toa(archive_entry_size(entry)));

410

}

411

if (w + strlen(tmp) >= bsdtar->gs_width)

412

bsdtar->gs_width = w+strlen(tmp)+1;

413

fprintf(out, "%*s", (int)(bsdtar->gs_width - w), tmp);

414

415

/* Format the time using 'ls -l' conventions. */

416

tim = archive_entry_mtime(entry);

417

#define HALF_YEAR (time_t)365 * 86400 / 2

418

#if defined(_WIN32) && !defined(__CYGWIN__)

419

#define DAY_FMT "%d" /* Windows' strftime function does not support %e format. */

420

#else

421

#define DAY_FMT "%e" /* Day number without leading zeros */

422

#endif

423

if (tim < now - HALF_YEAR || tim > now + HALF_YEAR)

424

fmt = bsdtar->day_first ? DAY_FMT " %b %Y" : "%b " DAY_FMT " %Y";

425

else

426

fmt = bsdtar->day_first ? DAY_FMT " %b %H:%M" : "%b " DAY_FMT " %H:%M";

427

strftime(tmp, sizeof(tmp), fmt, localtime(&tim));

428

fprintf(out, " %s ", tmp);

429

safe_fprintf(out, "%s", archive_entry_pathname(entry));

430

431

/* Extra information for links. */

432

if (archive_entry_hardlink(entry)) /* Hard link */

433

safe_fprintf(out, " link to %s",

434

archive_entry_hardlink(entry));

435

else if (archive_entry_symlink(entry)) /* Symbolic link */

436

safe_fprintf(out, " -> %s", archive_entry_symlink(entry));

437

}

Older »