~akopytov/percona-xtrabackup/bug1166888-2.0

Viewing changes to src/libarchive/libarchive/archive_write_set_format_cpio.c

Committer: Alexey Kopytov
Date: 2012-02-10 20:05:56 UTC
mto: This revision was merged to the branch mainline in revision 390.
Revision ID: akopytov@gmail.com-20120210200556-6kx41z8wwrqfucro

Rebase of the parallel compression patch on new trunk + post-review
fixes.

Implementation of parallel compression and streaming for XtraBackup.

This revision implements the following changes:

* InnoDB files are now streamed by the xtrabackup binary rather than
innobackupex. As a result, integrity is now verified by xtrabackup and
thus tar4ibd is no longer needed, so it was removed.

* xtrabackup binary now accepts the new '--stream' option which has
exactly the same semantics as the '--stream' option in
innobackupex: it tells xtrabackup to stream all files to the standard
output in the specified format rather than storing them locally.

* The xtrabackup binary can now do parallel compression using the
quicklz library. Two new options were added to xtrabackup to support
this feature:

- '--compress' tells xtrabackup to compress all output data, including
the transaction log file and meta data files, using the specified
compression algorithm. The only currently supported algorithm is
'quicklz'. The resulting files have the qpress archive format,
i.e. every *.qp file produced by xtrabackup is essentially a one-file
qpress archive and can be extracted and uncompressed by the qpress
file archiver (http://www.quicklz.com/).

- '--compress-threads' specifies the number of worker threads used by
xtrabackup for parallel data compression. This option defaults to 1.

Parallel compression ('--compress-threads') can be used together with
parallel file copying ('--parallel'). For example, '--parallel=4
--compress --compress-threads=2' will create 4 IO threads that will
read the data and pipe it to 2 compression threads.

* To support simultaneous compression and streaming, a new custom
streaming format called 'xbstream' was introduced to XtraBackup in
addition to the 'tar' format. That was required to overcome some
limitations of traditional archive formats such as 'tar', 'cpio' and
others that do not allow streaming dynamically generated files, for
example dynamically compressed files. Other advantages of xbstream over
traditional streaming/archive formats include ability to stream multiple
files concurrently (so it is possible to use streaming in the xbstream
format together with the --parallel option) and more compact data
storage.

* To allow streaming and extracting files to/from the xbstream format
produced by xtrabackup, a new utility aptly called 'xbstream' was
added to the XtraBackup distribution. This utility has a tar-like
interface:

- with the '-x' option it extracts files from the stream read from its
standard input to the current directory unless specified otherwise
with the '-C' option.

- with the '-c' option it streams files specified on the command line
to its standard output.

The utility also tries to minimize its impact on the OS page cache by
using the appropriate posix_fadvise() calls when available.

files added:
src

src/common.h

src/compress.c

src/compress.h

src/datasink.h

src/libarchive

src/libarchive/CMakeLists.txt

src/libarchive/COPYING

src/libarchive/INSTALL

src/libarchive/Makefile.am

src/libarchive/NEWS

src/libarchive/README

src/libarchive/build

src/libarchive/build/autoconf

src/libarchive/build/autoconf/check_stdcall_func.m4

src/libarchive/build/autoconf/compile

src/libarchive/build/autoconf/config.guess

src/libarchive/build/autoconf/config.sub

src/libarchive/build/autoconf/depcomp

src/libarchive/build/autoconf/install-sh

src/libarchive/build/autoconf/la_uid_t.m4

src/libarchive/build/autoconf/ltmain.sh

src/libarchive/build/autoconf/missing

src/libarchive/build/autogen.sh

src/libarchive/build/bump-version.sh

src/libarchive/build/clean.sh

src/libarchive/build/cmake

src/libarchive/build/cmake/AddTest28.cmake

src/libarchive/build/cmake/CheckFileOffsetBits.c

src/libarchive/build/cmake/CheckFileOffsetBits.cmake

src/libarchive/build/cmake/CheckFuncs.cmake

src/libarchive/build/cmake/CheckFuncs_stub.c.in

src/libarchive/build/cmake/CheckHeaderDirent.cmake

src/libarchive/build/cmake/CheckStructMember.cmake

src/libarchive/build/cmake/CheckTypeExists.cmake

src/libarchive/build/cmake/FindLZMA.cmake

src/libarchive/build/cmake/config.h.in

src/libarchive/build/pkgconfig

src/libarchive/build/pkgconfig/libarchive.pc.in

src/libarchive/build/version

src/libarchive/build/windows

src/libarchive/build/windows/mvcpp.nt

src/libarchive/build/windows/vc71

src/libarchive/build/windows/vc71/libarchive.sln

src/libarchive/build/windows/vc71/libarchive.vcproj

src/libarchive/build/windows/vc80

src/libarchive/build/windows/vc80/libarchive.sln

src/libarchive/build/windows/vc80/libarchive.vcproj

src/libarchive/build/windows/vc80/libarchive_test

src/libarchive/build/windows/vc80/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/vc90

src/libarchive/build/windows/vc90/libarchive.sln

src/libarchive/build/windows/vc90/libarchive.vcproj

src/libarchive/build/windows/vc90/libarchive_test

src/libarchive/build/windows/vc90/libarchive_test/libarchive_test.vcproj

src/libarchive/build/windows/wccpp.nt

src/libarchive/config.h.in

src/libarchive/configure.ac

src/libarchive/contrib

src/libarchive/contrib/README

src/libarchive/contrib/libarchive.1aix53.spec

src/libarchive/contrib/libarchive.spec

src/libarchive/contrib/libarchive_autodetect-st_lib_archive.m4

src/libarchive/contrib/psota-benchmark

src/libarchive/contrib/psota-benchmark/results.txt

src/libarchive/contrib/psota-benchmark/tcp.sh

src/libarchive/contrib/shar

src/libarchive/contrib/shar/shar.1

src/libarchive/contrib/shar/shar.c

src/libarchive/contrib/shar/tree.c

src/libarchive/contrib/shar/tree.h

src/libarchive/contrib/shar/tree_config.h

src/libarchive/contrib/untar.c

src/libarchive/cpio

src/libarchive/cpio/CMakeLists.txt

src/libarchive/cpio/bsdcpio.1

src/libarchive/cpio/cmdline.c

src/libarchive/cpio/config_freebsd.h

src/libarchive/cpio/cpio.c

src/libarchive/cpio/cpio.h

src/libarchive/cpio/cpio_platform.h

src/libarchive/cpio/cpio_windows.c

src/libarchive/cpio/cpio_windows.h

src/libarchive/cpio/test

src/libarchive/cpio/test/CMakeLists.txt

src/libarchive/cpio/test/main.c

src/libarchive/cpio/test/test.h

src/libarchive/cpio/test/test_0.c

src/libarchive/cpio/test/test_basic.c

src/libarchive/cpio/test/test_cmdline.c

src/libarchive/cpio/test/test_format_newc.c

src/libarchive/cpio/test/test_gcpio_compat.c

src/libarchive/cpio/test/test_gcpio_compat_ref.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref.ustar.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.bin.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.crc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.newc.uu

src/libarchive/cpio/test/test_gcpio_compat_ref_nosym.ustar.uu

src/libarchive/cpio/test/test_option_B_upper.c

src/libarchive/cpio/test/test_option_C_upper.c

src/libarchive/cpio/test/test_option_J_upper.c

src/libarchive/cpio/test/test_option_L_upper.c

src/libarchive/cpio/test/test_option_Z_upper.c

src/libarchive/cpio/test/test_option_a.c

src/libarchive/cpio/test/test_option_c.c

src/libarchive/cpio/test/test_option_d.c

src/libarchive/cpio/test/test_option_f.c

src/libarchive/cpio/test/test_option_f.cpio.uu

src/libarchive/cpio/test/test_option_help.c

src/libarchive/cpio/test/test_option_l.c

src/libarchive/cpio/test/test_option_lzma.c

src/libarchive/cpio/test/test_option_m.c

src/libarchive/cpio/test/test_option_m.cpio.uu

src/libarchive/cpio/test/test_option_t.c

src/libarchive/cpio/test/test_option_t.cpio.uu

src/libarchive/cpio/test/test_option_t.stdout.uu

src/libarchive/cpio/test/test_option_tv.stdout.uu

src/libarchive/cpio/test/test_option_u.c

src/libarchive/cpio/test/test_option_version.c

src/libarchive/cpio/test/test_option_y.c

src/libarchive/cpio/test/test_option_z.c

src/libarchive/cpio/test/test_owner_parse.c

src/libarchive/cpio/test/test_passthrough_dotdot.c

src/libarchive/cpio/test/test_passthrough_reverse.c

src/libarchive/cpio/test/test_pathmatch.c

src/libarchive/doc

src/libarchive/doc/html

src/libarchive/doc/man

src/libarchive/doc/mdoc2man.awk

src/libarchive/doc/mdoc2wiki.awk

src/libarchive/doc/pdf

src/libarchive/doc/text

src/libarchive/doc/update.sh

src/libarchive/doc/wiki

src/libarchive/examples

src/libarchive/examples/minitar

src/libarchive/examples/minitar/README

src/libarchive/examples/minitar/minitar.c

src/libarchive/examples/minitar/tree.c

src/libarchive/examples/minitar/tree.h

src/libarchive/examples/tarfilter.c

src/libarchive/examples/untar.c

src/libarchive/libarchive

src/libarchive/libarchive/CMakeLists.txt

src/libarchive/libarchive/archive.h

src/libarchive/libarchive/archive_check_magic.c

src/libarchive/libarchive/archive_crc32.h

src/libarchive/libarchive/archive_endian.h

src/libarchive/libarchive/archive_entry.3

src/libarchive/libarchive/archive_entry.c

src/libarchive/libarchive/archive_entry.h

src/libarchive/libarchive/archive_entry_copy_bhfi.c

src/libarchive/libarchive/archive_entry_copy_stat.c

src/libarchive/libarchive/archive_entry_link_resolver.c

src/libarchive/libarchive/archive_entry_private.h

src/libarchive/libarchive/archive_entry_stat.c

src/libarchive/libarchive/archive_entry_strmode.c

src/libarchive/libarchive/archive_entry_xattr.c

src/libarchive/libarchive/archive_hash.h

src/libarchive/libarchive/archive_platform.h

src/libarchive/libarchive/archive_private.h

src/libarchive/libarchive/archive_read.3

src/libarchive/libarchive/archive_read.c

src/libarchive/libarchive/archive_read_data_into_fd.c

src/libarchive/libarchive/archive_read_disk.3

src/libarchive/libarchive/archive_read_disk.c

src/libarchive/libarchive/archive_read_disk_entry_from_file.c

src/libarchive/libarchive/archive_read_disk_private.h

src/libarchive/libarchive/archive_read_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_read_extract.c

src/libarchive/libarchive/archive_read_open_fd.c

src/libarchive/libarchive/archive_read_open_file.c

src/libarchive/libarchive/archive_read_open_filename.c

src/libarchive/libarchive/archive_read_open_memory.c

src/libarchive/libarchive/archive_read_private.h

src/libarchive/libarchive/archive_read_support_compression_all.c

src/libarchive/libarchive/archive_read_support_compression_bzip2.c

src/libarchive/libarchive/archive_read_support_compression_compress.c

src/libarchive/libarchive/archive_read_support_compression_gzip.c

src/libarchive/libarchive/archive_read_support_compression_none.c

src/libarchive/libarchive/archive_read_support_compression_program.c

src/libarchive/libarchive/archive_read_support_compression_rpm.c

src/libarchive/libarchive/archive_read_support_compression_uu.c

src/libarchive/libarchive/archive_read_support_compression_xz.c

src/libarchive/libarchive/archive_read_support_format_all.c

src/libarchive/libarchive/archive_read_support_format_ar.c

src/libarchive/libarchive/archive_read_support_format_cpio.c

src/libarchive/libarchive/archive_read_support_format_empty.c

src/libarchive/libarchive/archive_read_support_format_iso9660.c

src/libarchive/libarchive/archive_read_support_format_mtree.c

src/libarchive/libarchive/archive_read_support_format_raw.c

src/libarchive/libarchive/archive_read_support_format_tar.c

src/libarchive/libarchive/archive_read_support_format_xar.c

src/libarchive/libarchive/archive_read_support_format_zip.c

src/libarchive/libarchive/archive_string.c

src/libarchive/libarchive/archive_string.h

src/libarchive/libarchive/archive_string_sprintf.c

src/libarchive/libarchive/archive_util.3

src/libarchive/libarchive/archive_util.c

src/libarchive/libarchive/archive_virtual.c

src/libarchive/libarchive/archive_windows.c

src/libarchive/libarchive/archive_windows.h

src/libarchive/libarchive/archive_write.3

src/libarchive/libarchive/archive_write.c

src/libarchive/libarchive/archive_write_disk.3

src/libarchive/libarchive/archive_write_disk.c

src/libarchive/libarchive/archive_write_disk_private.h

src/libarchive/libarchive/archive_write_disk_set_standard_lookup.c

src/libarchive/libarchive/archive_write_open_fd.c

src/libarchive/libarchive/archive_write_open_file.c

src/libarchive/libarchive/archive_write_open_filename.c

src/libarchive/libarchive/archive_write_open_memory.c

src/libarchive/libarchive/archive_write_private.h

src/libarchive/libarchive/archive_write_set_compression_bzip2.c

src/libarchive/libarchive/archive_write_set_compression_compress.c

src/libarchive/libarchive/archive_write_set_compression_gzip.c

src/libarchive/libarchive/archive_write_set_compression_none.c

src/libarchive/libarchive/archive_write_set_compression_program.c

src/libarchive/libarchive/archive_write_set_compression_xz.c

src/libarchive/libarchive/archive_write_set_format.c

src/libarchive/libarchive/archive_write_set_format_ar.c

src/libarchive/libarchive/archive_write_set_format_by_name.c

src/libarchive/libarchive/archive_write_set_format_cpio.c

src/libarchive/libarchive/archive_write_set_format_cpio_newc.c

src/libarchive/libarchive/archive_write_set_format_mtree.c

src/libarchive/libarchive/archive_write_set_format_pax.c

src/libarchive/libarchive/archive_write_set_format_shar.c

src/libarchive/libarchive/archive_write_set_format_ustar.c

src/libarchive/libarchive/archive_write_set_format_zip.c

src/libarchive/libarchive/config_freebsd.h

src/libarchive/libarchive/cpio.5

src/libarchive/libarchive/filter_fork.c

src/libarchive/libarchive/filter_fork.h

src/libarchive/libarchive/filter_fork_windows.c

src/libarchive/libarchive/libarchive-formats.5

src/libarchive/libarchive/libarchive.3

src/libarchive/libarchive/libarchive_internals.3

src/libarchive/libarchive/mtree.5

src/libarchive/libarchive/tar.5

src/libarchive/libarchive/test

src/libarchive/libarchive/test/CMakeLists.txt

src/libarchive/libarchive/test/README

src/libarchive/libarchive/test/main.c

src/libarchive/libarchive/test/read_open_memory.c

src/libarchive/libarchive/test/test.h

src/libarchive/libarchive/test/test_acl_basic.c

src/libarchive/libarchive/test/test_acl_freebsd.c

src/libarchive/libarchive/test/test_acl_pax.c

src/libarchive/libarchive/test/test_archive_api_feature.c

src/libarchive/libarchive/test/test_bad_fd.c

src/libarchive/libarchive/test/test_compat_bzip2.c

src/libarchive/libarchive/test/test_compat_bzip2_1.tbz.uu

src/libarchive/libarchive/test/test_compat_bzip2_2.tbz.uu

src/libarchive/libarchive/test/test_compat_cpio.c

src/libarchive/libarchive/test/test_compat_cpio_1.cpio.uu

src/libarchive/libarchive/test/test_compat_gtar.c

src/libarchive/libarchive/test/test_compat_gtar_1.tar.uu

src/libarchive/libarchive/test/test_compat_gzip.c

src/libarchive/libarchive/test/test_compat_gzip_1.tgz.uu

src/libarchive/libarchive/test/test_compat_gzip_2.tgz.uu

src/libarchive/libarchive/test/test_compat_lzma.c

src/libarchive/libarchive/test/test_compat_lzma_1.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_2.tlz.uu

src/libarchive/libarchive/test/test_compat_lzma_3.tlz.uu

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.c

src/libarchive/libarchive/test/test_compat_solaris_tar_acl.tar.uu

src/libarchive/libarchive/test/test_compat_tar_hardlink.c

src/libarchive/libarchive/test/test_compat_tar_hardlink_1.tar.uu

src/libarchive/libarchive/test/test_compat_xz.c

src/libarchive/libarchive/test/test_compat_xz_1.txz.uu

src/libarchive/libarchive/test/test_compat_zip.c

src/libarchive/libarchive/test/test_compat_zip_1.zip.uu

src/libarchive/libarchive/test/test_empty_write.c

src/libarchive/libarchive/test/test_entry.c

src/libarchive/libarchive/test/test_entry_strmode.c

src/libarchive/libarchive/test/test_extattr_freebsd.c

src/libarchive/libarchive/test/test_fuzz.c

src/libarchive/libarchive/test/test_fuzz_1.iso.Z.uu

src/libarchive/libarchive/test/test_link_resolver.c

src/libarchive/libarchive/test/test_open_fd.c

src/libarchive/libarchive/test/test_open_file.c

src/libarchive/libarchive/test/test_open_filename.c

src/libarchive/libarchive/test/test_pax_filename_encoding.c

src/libarchive/libarchive/test/test_pax_filename_encoding.tar.uu

src/libarchive/libarchive/test/test_read_compress_program.c

src/libarchive/libarchive/test/test_read_data_large.c

src/libarchive/libarchive/test/test_read_disk.c

src/libarchive/libarchive/test/test_read_disk_entry_from_file.c

src/libarchive/libarchive/test/test_read_extract.c

src/libarchive/libarchive/test/test_read_file_nonexistent.c

src/libarchive/libarchive/test/test_read_format_ar.ar.uu

src/libarchive/libarchive/test/test_read_format_ar.c

src/libarchive/libarchive/test/test_read_format_cpio_bin.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_Z.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_be.cpio.uu

src/libarchive/libarchive/test/test_read_format_cpio_bin_bz2.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_gz.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_lzma.c

src/libarchive/libarchive/test/test_read_format_cpio_bin_xz.c

src/libarchive/libarchive/test/test_read_format_cpio_odc.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_bzip2_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.c

src/libarchive/libarchive/test/test_read_format_cpio_svr4_gzip_rpm.rpm.uu

src/libarchive/libarchive/test/test_read_format_cpio_svr4c_Z.c

src/libarchive/libarchive/test/test_read_format_empty.c

src/libarchive/libarchive/test/test_read_format_gtar_gz.c

src/libarchive/libarchive/test/test_read_format_gtar_lzma.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse.c

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_13.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix00.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix01.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10.tar.uu

src/libarchive/libarchive/test/test_read_format_gtar_sparse_1_17_posix10_modified.tar.uu

src/libarchive/libarchive/test/test_read_format_iso.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_gz.c

src/libarchive/libarchive/test/test_read_format_iso_joliet.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_long.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_joliet_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.c

src/libarchive/libarchive/test/test_read_format_iso_multi_extent.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_ce.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_new.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_rockridge_rr_moved.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_iso_zisofs.iso.Z.uu

src/libarchive/libarchive/test/test_read_format_isojoliet_bz2.c

src/libarchive/libarchive/test/test_read_format_isojoliet_long.c

src/libarchive/libarchive/test/test_read_format_isojoliet_rr.c

src/libarchive/libarchive/test/test_read_format_isorr_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_ce.c

src/libarchive/libarchive/test/test_read_format_isorr_new_bz2.c

src/libarchive/libarchive/test/test_read_format_isorr_rr_moved.c

src/libarchive/libarchive/test/test_read_format_isozisofs_bz2.c

src/libarchive/libarchive/test/test_read_format_mtree.c

src/libarchive/libarchive/test/test_read_format_mtree.mtree.uu

src/libarchive/libarchive/test/test_read_format_pax_bz2.c

src/libarchive/libarchive/test/test_read_format_raw.c

src/libarchive/libarchive/test/test_read_format_raw.data.Z.uu

src/libarchive/libarchive/test/test_read_format_raw.data.uu

src/libarchive/libarchive/test/test_read_format_tar.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.c

src/libarchive/libarchive/test/test_read_format_tar_empty_filename.tar.uu

src/libarchive/libarchive/test/test_read_format_tbz.c

src/libarchive/libarchive/test/test_read_format_tgz.c

src/libarchive/libarchive/test/test_read_format_tlz.c

src/libarchive/libarchive/test/test_read_format_txz.c

src/libarchive/libarchive/test/test_read_format_tz.c

src/libarchive/libarchive/test/test_read_format_xar.c

src/libarchive/libarchive/test/test_read_format_zip.c

src/libarchive/libarchive/test/test_read_format_zip.zip.uu

src/libarchive/libarchive/test/test_read_large.c

src/libarchive/libarchive/test/test_read_pax_truncated.c

src/libarchive/libarchive/test/test_read_position.c

src/libarchive/libarchive/test/test_read_truncated.c

src/libarchive/libarchive/test/test_read_uu.c

src/libarchive/libarchive/test/test_tar_filenames.c

src/libarchive/libarchive/test/test_tar_large.c

src/libarchive/libarchive/test/test_ustar_filenames.c

src/libarchive/libarchive/test/test_write_compress.c

src/libarchive/libarchive/test/test_write_compress_bzip2.c

src/libarchive/libarchive/test/test_write_compress_gzip.c

src/libarchive/libarchive/test/test_write_compress_lzma.c

src/libarchive/libarchive/test/test_write_compress_program.c

src/libarchive/libarchive/test/test_write_compress_xz.c

src/libarchive/libarchive/test/test_write_disk.c

src/libarchive/libarchive/test/test_write_disk_failures.c

src/libarchive/libarchive/test/test_write_disk_hardlink.c

src/libarchive/libarchive/test/test_write_disk_perms.c

src/libarchive/libarchive/test/test_write_disk_secure.c

src/libarchive/libarchive/test/test_write_disk_sparse.c

src/libarchive/libarchive/test/test_write_disk_symlink.c

src/libarchive/libarchive/test/test_write_disk_times.c

src/libarchive/libarchive/test/test_write_format_ar.c

src/libarchive/libarchive/test/test_write_format_cpio.c

src/libarchive/libarchive/test/test_write_format_cpio_empty.c

src/libarchive/libarchive/test/test_write_format_cpio_newc.c

src/libarchive/libarchive/test/test_write_format_cpio_odc.c

src/libarchive/libarchive/test/test_write_format_mtree.c

src/libarchive/libarchive/test/test_write_format_pax.c

src/libarchive/libarchive/test/test_write_format_shar_empty.c

src/libarchive/libarchive/test/test_write_format_tar.c

src/libarchive/libarchive/test/test_write_format_tar_empty.c

src/libarchive/libarchive/test/test_write_format_tar_ustar.c

src/libarchive/libarchive/test/test_write_format_zip.c

src/libarchive/libarchive/test/test_write_format_zip_empty.c

src/libarchive/libarchive/test/test_write_format_zip_no_compression.c

src/libarchive/libarchive/test/test_write_open_memory.c

src/libarchive/libarchive_fe

src/libarchive/libarchive_fe/err.c

src/libarchive/libarchive_fe/err.h

src/libarchive/libarchive_fe/lafe_platform.h

src/libarchive/libarchive_fe/line_reader.c

src/libarchive/libarchive_fe/line_reader.h

src/libarchive/libarchive_fe/matching.c

src/libarchive/libarchive_fe/matching.h

src/libarchive/libarchive_fe/pathmatch.c

src/libarchive/libarchive_fe/pathmatch.h

src/libarchive/tar

src/libarchive/tar/CMakeLists.txt

src/libarchive/tar/bsdtar.1

src/libarchive/tar/bsdtar.c

src/libarchive/tar/bsdtar.h

src/libarchive/tar/bsdtar_platform.h

src/libarchive/tar/bsdtar_windows.c

src/libarchive/tar/bsdtar_windows.h

src/libarchive/tar/cmdline.c

src/libarchive/tar/config_freebsd.h

src/libarchive/tar/getdate.c

src/libarchive/tar/read.c

src/libarchive/tar/subst.c

src/libarchive/tar/test

src/libarchive/tar/test/CMakeLists.txt

src/libarchive/tar/test/main.c

src/libarchive/tar/test/test.h

src/libarchive/tar/test/test_0.c

src/libarchive/tar/test/test_basic.c

src/libarchive/tar/test/test_copy.c

src/libarchive/tar/test/test_empty_mtree.c

src/libarchive/tar/test/test_getdate.c

src/libarchive/tar/test/test_help.c

src/libarchive/tar/test/test_option_T_upper.c

src/libarchive/tar/test/test_option_q.c

src/libarchive/tar/test/test_option_r.c

src/libarchive/tar/test/test_option_s.c

src/libarchive/tar/test/test_patterns.c

src/libarchive/tar/test/test_patterns_2.tar.uu

src/libarchive/tar/test/test_patterns_3.tar.uu

src/libarchive/tar/test/test_patterns_4.tar.uu

src/libarchive/tar/test/test_stdio.c

src/libarchive/tar/test/test_strip_components.c

src/libarchive/tar/test/test_symlink_dir.c

src/libarchive/tar/test/test_version.c

src/libarchive/tar/test/test_windows.c

src/libarchive/tar/tree.c

src/libarchive/tar/tree.h

src/libarchive/tar/util.c

src/libarchive/tar/write.c

src/local.c

src/local.h

src/quicklz

src/quicklz/quicklz.c

src/quicklz/quicklz.h

src/stream.c

src/stream.h

src/xbstream.c

src/xbstream.h

src/xbstream_read.c

src/xbstream_write.c

test/t/ib_stream_compress.sh

test/t/ib_stream_parallel.sh

test/t/ib_stream_tar.sh

test/t/ib_stream_xbstream.sh

files removed:
patches/tar4ibd_libtar-1.2.11.patch

files renamed:
Makefile => src/Makefile

xb_regex.h => src/xb_regex.h

xtrabackup.c => src/xtrabackup.c

test/t/xb_stream.sh => test/inc/ib_stream_common.sh

files modified:
innobackupex

test/inc/common.sh

test/run.sh

test/t/bug514068.sh

test/t/bug606981.sh *

test/t/bug759225.sh

test/t/tar4ibd_symlink.sh

test/t/xb_incremental_compressed.sh

utils/build.sh

Show diffs side-by-side

added added

removed removed

src/libarchive/libarchive/archive_write_set_format_cpio.c

/*-

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR

* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "archive_platform.h"

__FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_format_cpio.c 201170 2009-12-29 06:34:23Z kientzle $");

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#include <stdio.h>

#ifdef HAVE_STDLIB_H

#include <stdlib.h>

#endif

#ifdef HAVE_STRING_H

#include <string.h>

#endif

#include "archive.h"

#include "archive_entry.h"

#include "archive_private.h"

#include "archive_write_private.h"

static ssize_t archive_write_cpio_data(struct archive_write *,

const void *buff, size_t s);

static int archive_write_cpio_finish(struct archive_write *);

static int archive_write_cpio_destroy(struct archive_write *);

static int archive_write_cpio_finish_entry(struct archive_write *);

static int archive_write_cpio_header(struct archive_write *,

struct archive_entry *);

static int format_octal(int64_t, void *, int);

static int64_t format_octal_recursive(int64_t, char *, int);

struct cpio {

uint64_t entry_bytes_remaining;

int64_t ino_next;

struct { int64_t old; int new;} *ino_list;

size_t ino_list_size;

size_t ino_list_next;

};

struct cpio_header {

char c_magic[6];

char c_dev[6];

char c_ino[6];

char c_mode[6];

char c_uid[6];

char c_gid[6];

char c_nlink[6];

char c_rdev[6];

char c_mtime[11];

char c_namesize[6];

char c_filesize[11];

};

* Set output format to 'cpio' format.

int

archive_write_set_format_cpio(struct archive *_a)

{

struct archive_write *a = (struct archive_write *)_a;

struct cpio *cpio;

/* If someone else was already registered, unregister them. */

if (a->format_destroy != NULL)

(a->format_destroy)(a);

cpio = (struct cpio *)malloc(sizeof(*cpio));

if (cpio == NULL) {

archive_set_error(&a->archive, ENOMEM, "Can't allocate cpio data");

return (ARCHIVE_FATAL);

}

memset(cpio, 0, sizeof(*cpio));

a->format_data = cpio;

100

a->pad_uncompressed = 1;

101

a->format_name = "cpio";

102

a->format_write_header = archive_write_cpio_header;

103

a->format_write_data = archive_write_cpio_data;

104

a->format_finish_entry = archive_write_cpio_finish_entry;

105

a->format_finish = archive_write_cpio_finish;

106

a->format_destroy = archive_write_cpio_destroy;

107

a->archive.archive_format = ARCHIVE_FORMAT_CPIO_POSIX;

108

a->archive.archive_format_name = "POSIX cpio";

109

return (ARCHIVE_OK);

110

}

111

112

113

* Ino values are as long as 64 bits on some systems; cpio format

114

* only allows 18 bits and relies on the ino values to identify hardlinked

115

* files. So, we can't merely "hash" the ino numbers since collisions

116

* would corrupt the archive. Instead, we generate synthetic ino values

117

* to store in the archive and maintain a map of original ino values to

118

* synthetic ones so we can preserve hardlink information.

119

120

* TODO: Make this more efficient. It's not as bad as it looks (most

121

* files don't have any hardlinks and we don't do any work here for those),

122

* but it wouldn't be hard to do better.

123

124

* TODO: Work with dev/ino pairs here instead of just ino values.

125

126

static int

127

synthesize_ino_value(struct cpio *cpio, struct archive_entry *entry)

128

{

129

int64_t ino = archive_entry_ino64(entry);

130

int ino_new;

131

size_t i;

132

133

134

* If no index number was given, don't assign one. In

135

* particular, this handles the end-of-archive marker

136

* correctly by giving it a zero index value. (This is also

137

* why we start our synthetic index numbers with one below.)

138

139

if (ino == 0)

140

return (0);

141

142

/* Don't store a mapping if we don't need to. */

143

if (archive_entry_nlink(entry) < 2) {

144

return ++cpio->ino_next;

145

}

146

147

/* Look up old ino; if we have it, this is a hardlink

148

* and we reuse the same value. */

149

for (i = 0; i < cpio->ino_list_next; ++i) {

150

if (cpio->ino_list[i].old == ino)

151

return (cpio->ino_list[i].new);

152

}

153

154

/* Assign a new index number. */

155

ino_new = ++cpio->ino_next;

156

157

/* Ensure space for the new mapping. */

158

if (cpio->ino_list_size <= cpio->ino_list_next) {

159

size_t newsize = cpio->ino_list_size < 512

160

? 512 : cpio->ino_list_size * 2;

161

void *newlist = realloc(cpio->ino_list,

162

sizeof(cpio->ino_list[0]) * newsize);

163

if (newlist == NULL)

164

return (-1);

165

166

cpio->ino_list_size = newsize;

167

cpio->ino_list = newlist;

168

}

169

170

/* Record and return the new value. */

171

cpio->ino_list[cpio->ino_list_next].old = ino;

172

cpio->ino_list[cpio->ino_list_next].new = ino_new;

173

++cpio->ino_list_next;

174

return (ino_new);

175

}

176

177

static int

178

archive_write_cpio_header(struct archive_write *a, struct archive_entry *entry)

179

{

180

struct cpio *cpio;

181

const char *p, *path;

182

int pathlength, ret, ret2;

183

int64_t ino;

184

struct cpio_header h;

185

186

cpio = (struct cpio *)a->format_data;

187

ret2 = ARCHIVE_OK;

188

189

path = archive_entry_pathname(entry);

190

pathlength = (int)strlen(path) + 1; /* Include trailing null. */

191

192

memset(&h, 0, sizeof(h));

193

format_octal(070707, &h.c_magic, sizeof(h.c_magic));

194

format_octal(archive_entry_dev(entry), &h.c_dev, sizeof(h.c_dev));

195

196

ino = synthesize_ino_value(cpio, entry);

197

if (ino < 0) {

198

archive_set_error(&a->archive, ENOMEM,

199

"No memory for ino translation table");

200

return (ARCHIVE_FATAL);

201

} else if (ino > 0777777) {

202

archive_set_error(&a->archive, ERANGE,

203

"Too many files for this cpio format");

204

return (ARCHIVE_FATAL);

205

}

206

format_octal(ino & 0777777, &h.c_ino, sizeof(h.c_ino));

207

208

format_octal(archive_entry_mode(entry), &h.c_mode, sizeof(h.c_mode));

209

format_octal(archive_entry_uid(entry), &h.c_uid, sizeof(h.c_uid));

210

format_octal(archive_entry_gid(entry), &h.c_gid, sizeof(h.c_gid));

211

format_octal(archive_entry_nlink(entry), &h.c_nlink, sizeof(h.c_nlink));

212

if (archive_entry_filetype(entry) == AE_IFBLK

213

|| archive_entry_filetype(entry) == AE_IFCHR)

214

format_octal(archive_entry_dev(entry), &h.c_rdev, sizeof(h.c_rdev));

215

else

216

format_octal(0, &h.c_rdev, sizeof(h.c_rdev));

217

format_octal(archive_entry_mtime(entry), &h.c_mtime, sizeof(h.c_mtime));

218

format_octal(pathlength, &h.c_namesize, sizeof(h.c_namesize));

219

220

/* Non-regular files don't store bodies. */

221

if (archive_entry_filetype(entry) != AE_IFREG)

222

archive_entry_set_size(entry, 0);

223

224

/* Symlinks get the link written as the body of the entry. */

225

p = archive_entry_symlink(entry);

226

if (p != NULL && *p != '\0')

227

format_octal(strlen(p), &h.c_filesize, sizeof(h.c_filesize));

228

else

229

format_octal(archive_entry_size(entry),

230

&h.c_filesize, sizeof(h.c_filesize));

231

232

ret = (a->compressor.write)(a, &h, sizeof(h));

233

if (ret != ARCHIVE_OK)

234

return (ARCHIVE_FATAL);

235

236

ret = (a->compressor.write)(a, path, pathlength);

237

if (ret != ARCHIVE_OK)

238

return (ARCHIVE_FATAL);

239

240

cpio->entry_bytes_remaining = archive_entry_size(entry);

241

242

/* Write the symlink now. */

243

if (p != NULL && *p != '\0')

244

ret = (a->compressor.write)(a, p, strlen(p));

245

246

if (ret == ARCHIVE_OK)

247

ret = ret2;

248

return (ret);

249

}

250

251

static ssize_t

252

archive_write_cpio_data(struct archive_write *a, const void *buff, size_t s)

253

{

254

struct cpio *cpio;

255

int ret;

256

257

cpio = (struct cpio *)a->format_data;

258

if (s > cpio->entry_bytes_remaining)

259

s = cpio->entry_bytes_remaining;

260

261

ret = (a->compressor.write)(a, buff, s);

262

cpio->entry_bytes_remaining -= s;

263

if (ret >= 0)

264

return (s);

265

else

266

return (ret);

267

}

268

269

270

* Format a number into the specified field.

271

272

static int

273

format_octal(int64_t v, void *p, int digits)

274

{

275

int64_t max;

276

int ret;

277

278

max = (((int64_t)1) << (digits * 3)) - 1;

279

if (v >= 0 && v <= max) {

280

format_octal_recursive(v, (char *)p, digits);

281

ret = 0;

282

} else {

283

format_octal_recursive(max, (char *)p, digits);

284

ret = -1;

285

}

286

return (ret);

287

}

288

289

static int64_t

290

format_octal_recursive(int64_t v, char *p, int s)

291

{

292

if (s == 0)

293

return (v);

294

v = format_octal_recursive(v, p+1, s-1);

295

*p = '0' + (v & 7);

296

return (v >> 3);

297

}

298

299

static int

300

archive_write_cpio_finish(struct archive_write *a)

301

{

302

int er;

303

struct archive_entry *trailer;

304

305

trailer = archive_entry_new();

306

/* nlink = 1 here for GNU cpio compat. */

307

archive_entry_set_nlink(trailer, 1);

308

archive_entry_set_pathname(trailer, "TRAILER!!!");

309

er = archive_write_cpio_header(a, trailer);

310

archive_entry_free(trailer);

311

return (er);

312

}

313

314

static int

315

archive_write_cpio_destroy(struct archive_write *a)

316

{

317

struct cpio *cpio;

318

319

cpio = (struct cpio *)a->format_data;

320

free(cpio->ino_list);

321

free(cpio);

322

a->format_data = NULL;

323

return (ARCHIVE_OK);

324

}

325

326

static int

327

archive_write_cpio_finish_entry(struct archive_write *a)

328

{

329

struct cpio *cpio;

330

size_t to_write;

331

int ret;

332

333

cpio = (struct cpio *)a->format_data;

334

ret = ARCHIVE_OK;

335

while (cpio->entry_bytes_remaining > 0) {

336

to_write = cpio->entry_bytes_remaining < a->null_length ?

337

cpio->entry_bytes_remaining : a->null_length;

338

ret = (a->compressor.write)(a, a->nulls, to_write);

339

if (ret != ARCHIVE_OK)

340

return (ret);

341

cpio->entry_bytes_remaining -= to_write;

342

}

343

return (ret);

344

}

Older »