~ubuntu-on-ec2/vmbuilder/automated-ec2-builds

#!/bin/bash

# vi: ts=4 noexpandtab

umask 022

cmd_args=$*

base_d=$(dirname $(readlink -f "${0}"))

SKIP_FINISH=0

serial=""

TEMP_D=""

error() { echo "$@" 1>&2; }

debug() { error "$(date -R):" "$@"; }

cleanup() {

		[ "${preserve_image}" = "1" -o -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"

}

cat_partfile() {

	# Note: this function may be overwritten in a config file

	local arch=${1} size=${2}

	if [ "${arch}" = "i386" ]; then

		printf "%s\n%s\n%s\n" "root ${size} a1" \

			"/mnt 1 a2" "swap 1024 a3"

	else

		printf "%s\n%s\n" "root ${size} a1" "/mnt 1 b"

	fi

}

fail() { [ $# -eq 0 ] || error "$@"; exit 1; }

Usage() {

	cat <<EOF

Usage: ${0##*/} config output_d

   build according to config, putting output in output_d

EOF

}

bad_Usage() { Usage 1>&2; fail "$@"; }

# optional_file(filename,directory_src,target)

#  if filename is non-empty, copy it from directory_src to target

#  return in _RET the target

#  if filename is empty, return true, but _RET as empty.

optional_file() {

	local fname="$1" src_d=${2} targ=${3}

	[ -n "${fname}" ] || { _RET=""; return 0; }

	cp "${src_d}/${fname}" "${targ}" && _RET=${targ} && return 0

	return 1

}

short_opts="h"

long_opts="help,skip-finish,serial:,proposed,hwe"

getopt_out=$(getopt --name "${0##*/}" \

	--options "${short_opts}" --long "${long_opts}" -- "$@") &&

	eval set -- "${getopt_out}" ||

	bad_Usage

skip_gen_image=""

preserve_image=0

proposed=0

hwe=0

while [ $# -ne 0 ]; do

	cur=${1}; next=${2};

	case "$cur" in

		-h|--help) Usage; exit 0;;

		   --serial) serial=$2; shift;;

		   --skip-finish) SKIP_FINISH=1;;

		   --image) skip_gen_image=$1;;

		   --preserve) preserve_image=1;;

		   --proposed) proposed=1;;

		   --hwe) hwe=1;;

		--) shift; break;;

	esac

	shift;

done

[ $# -eq 2 ] || bad_Usage "must supply config and output dir"

conf=${1}

pub_d=${2}

[ -f "${conf}" ] || fail "${conf}: not a file"

owner=${USER}

suite=""

build_name=""

description=""

copy_out_kernels=1

unpublished_ramdisks=""

publish_ec2=1

fail_on_missing_kernels=1

root_fs_size="2048"

root_fs_label=""

loader=""

floppy=""

ovf_create=0

ovf_cmd=( )

ova_create=0

ova_cmd=( )

package_img_tgz=1

create_root_tgz=0

purge_roottgz_pkgs=( )

. "${conf}"

[ -n "${arches}" ] || arches="${img_arches}"

for x in suite build_name description; do

	[ -n "${!x}" ] || fail "${x} is not defined in ${conf}"

done

VMBUILDER=$(which vmbuilder 2>/dev/null) || fail "cannot find vmbuilder"

LBBUILD=$(which build-lb-ec2-image 2> /dev/null ) || fail "cannont find build-lb-ec2-image"

BUILDD_BUILD=$(which build-buildd-image 2> /dev/null) || fail "cannot find build-buildd-image"

{ [ -d "${pub_d}" ] || mkdir -p "${pub_d}"; } ||

	fail "failed to make ${pub_d}"

TEMP_D=$(mktemp -d ${pub_d}/.${0##*/}.tmp.XXXXXX) ||

	fail "failed to make tempd"

trap cleanup EXIT

chownperms=$(printf "%s:%s" "$(id -u)" "$(id -g)")

unpacked_d="${pub_d}/unpacked"

mkdir -p "${unpacked_d}" || fail "failed to make ${unpacked_d}"

ovf_d="${pub_d}"

if [ ${ovf_create:-0} -ne 0 -o ${ova_create:-0} -ne 1 ]; then

	mkdir -p "${ovf_d}" || fail "failed to make ovf/ova dir ${ovf_d}"

fi

for arch in ${arches}

do

	name_prefix="${suite}-${build_name}-cloudimg"

	img_base="${name_prefix}-${arch}"

	tmpdest="${pub_d}/tmp-${img_base}"

	# set variables read by copy-out-kernels

	klist=""

	partfile=${TEMP_D}/partfile.${arch}

	cat_partfile "${arch}" "${root_fs_size}" > "${partfile}" ||

		fail "failed to write partfile: cat_partfile $arch $root_fs_size"

	# expose certain variables to the execscript

	# this could also be done through sudo -E, but this just

	# seems more reliable

	execscript="${TEMP_D}/execscript"

	{

		spin=

		printf "#!/bin/bash\n"

		for v in suite serial build_name root_fs_label; do

			printf "%s='%s'\n" "$v" "${!v}"

		done

		printf "\n"

		cat "${base_d}/vmbuilder-cloudimg-fixes"

	} > "${execscript}"

	chmod 755 "${execscript}"

	kinfo="${img_base}-kernel-info.txt"

	image="${img_base}.img"

	# Use old scripts for Natty and older, and new for Oneiric and later

	if [ "${suite}" \< "oneiric" -o "${BUILDER}" = "vmbuilder" ]

	then

		cmdline=(

		   ${VMBUILDER} xen ubuntu

		   "--suite=${suite}"

		   "--arch=${arch}"

		   "--dest=${tmpdest}"

		   --ec2

		   "--ec2-version=${description}"

		   --components main,restricted,universe,multiverse

		   --part "${partfile}"

		   --ec2-landscape --lock-user

		   "--manifest=${tmpdest}/${img_base}.manifest"

		   "--tmp=${TMPDIR:-/tmp}"

		   --execscript "${execscript}"

		   --debug "${vmb_addargs[@]}" )

	elif [ "${suite}" \< "v" -o "${BUILDER}" = "live-build" ]

	then

		cmdline=(

			${LBBUILD}

			"--suite=${suite}"

			"--arch=${arch}"

			"--dest=${tmpdest}"

			"--image=root.img"

			"--part=${partfile}"

			"--rootlabel=${root_fs_label}"

			"--rootsize=${root_fs_size}"

			"--owner=${owner}"

			"--manifest=${tmpdest}/${img_base}.manifest"

			"--livepath=${LIVE_BUILD_PATH}"

			"--serial=${serial}"

			"--hookscript=${execscript}"

			"--ec2-version=${description}"

			"--type=${build_type:-server}"

			"--config=${conf}"

		 )

	elif [ "${suite}" \> "u" -o "${BUILDER}" = "buildd" ]

	then

		cmdline=(

			${BUILDD_BUILD}

			"--suite=${suite}"

			"--arch=${arch}"

			"--dest=${tmpdest}"

			"--image=root.img"

			"--owner=${owner}"

			"--manifest=${tmpdest}/${img_base}.manifest"

		 )

	fi

	# Allow building from propopsed

	[ "${proposed:-0}" -eq 1 ] && cmdline+=('--proposed')

	debug "invoking ${cmdline[*]}"

	if [ -z ${skip_gen_image} ]; then

		sudo env "PATH=$PATH" "${cmdline[@]}" ||

			fail "image generation command failed: ${cmdline[*]}";

	else

		sudo cp ${skip_gen_image} ${tmpdest} ||

			fail "unable to copy ${skip_gen_image} to ${tmpdest}";

	fi

	if [ "${suite}" \> "v" -o "${BUILDER}" = "buildd" ]; then

		sudo env "PATH=$PATH" mount-image-callback "${tmpdest}/root.img" -- \

			write-versions _MOUNTPOINT_ "${build_type:-server}" "${serial}" \

			"${description}"

	fi

	if [ ${copy_out_kernels} -ne 0 ]; then

		debug "${img_base}: copying out kernels"

		sudo env "PATH=$PATH" mount-image-callback \

			--read-only "${tmpdest}/root.img" -- copy-out-kernels \

			${unpublished_ramdisks:+"--skip-ramdisks=${unpublished_ramdisks}"} \

			--chown ${chownperms} \

			_MOUNTPOINT_ "${tmpdest}" \

			"${tmpdest}/${kinfo}" "${img_base}-" ||

			fail "failed copy out kernels"

		[ "$fail_on_missing_kernels" = "0" -o -s "${tmpdest}/${kinfo}" ] ||

			fail "no kernels found"

	fi

	if [ ${create_root_tgz} -ne 0 ]; then

		debug "${img_base}: creating root tarball"

		rpurgeopts=()

		for purgeopt in "${purge_roottgz_pkgs[@]}"; do

			rpurgeopts[${#rpurgeopts[@]}]="--purge=${purgeopt}"

		done

		if [ "${#rpurgeopts[@]}" -eq 0 ]; then

			# img2roottgz will mount 'ro' if no packages given

			ln -sf "$tmpdest/root.img" "$tmpdest/root.img.copy" ||

				fail "oddly failed to create symlink to $root.img.copy"

		else

			cp --sparse=always "${tmpdest}/root.img" \

				"${tmpdest}/root.img.copy" ||

			fail "failed copy of root img before img2roottgz"

		fi

		sudo env "PATH=$PATH" img2roottgz "${rpurgeopts[@]}" \

			"${tmpdest}/root.img.copy" "${pub_d}/${img_base}-root.tar.gz" ||

			fail "failed to run img2roottgz"

		rm -f "$tmpdest/root.img.copy"

		gzip -dc "${pub_d}/${img_base}-root.tar.gz" | 

			xz > "${pub_d}/${img_base}-root.tar.xz" ||

			fail "failed to create xz-compressed root.tar"

		sudo env PATH=$PATH serial=$serial arch=$arch suite=$suite \

			lxd_metadata "${pub_d}/${img_base}-lxd.tar.xz" ||

			fail "failed to create LXD metadata for root.tar"

	fi

	rootimg="${tmpdest}/root.img"

	# Zero out the free space, to make the image smaller

	zerofree ${rootimg}

	sync

	if [ -n "${root_fs_label}" ]; then

		e2label "${rootimg}" "${root_fs_label}" ||

			fail "failed to add label ${root_fs_label} to ${rootimg}"

	fi

	debug "${img_base}: moving files produced by vmbuilder"

	mv "${rootimg}" "${tmpdest}/${img_base}.img" ||

		fail "failed to move root.img to ${img_base}"

	[ "${SKIP_FINISH:-0}" = "0" ] || continue

	# move all files to 'unpacked' dir

	mv ${tmpdest}/${img_base}*  "${unpacked_d}/" ||

		fail "failed to move files to ${unpacked_d}"

	addl_tarmembers=""

	opt_ldrs=""

	# 'loader' could have a '%a' in it which would be replaced by arch

	optional_file "${loader//%a/${arch}}" "${base_d}/loaders" \

		"${unpacked_d}/${img_base}-loader" &&

		{ [ -z "${_RET}" ] || opt_ldrs="${opt_ldrs} ${_RET##*/}"; } ||

		fail "failed to copy optional loader"

	optional_file "${floppy//%a/${arch}}" "${base_d}/loaders" \

		"${unpacked_d}/${img_base}-floppy" &&

		{ [ -z "${_RET}" ] || opt_ldrs="${opt_ldrs} ${_RET##*/}"; } ||

		fail "failed to copy optional floppy"

	# get a list of non-ec2 kernels in unpacked_d

	if [ -s "${unpacked_d}/${kinfo}" -o -n "${opt_ldrs}" ]; then

		addl_tarmembers=$( cd "${unpacked_d}" ; out="";

			for f in ${img_base}*initrd* ${img_base}*vmlinu* ${opt_ldrs}; do

				[ -f "${f}" -a "${f##*-ec2}" = "${f}" ] || continue

				out="${out} ${f}"

			done

			[ -z "${out}" ] || echo "${out}"

		)

	fi

	debug "${img_base}: creating tarfiles"

	( cd "${unpacked_d}" && tar \

		-S --use-compress-program=gzip-rsyncable \

		-cf "${pub_d}/${img_base}.tar.gz" "${image}" ${addl_tarmembers} \

		-C "${base_d}" README.files ) ||

		fail "failed to create ${pub_d}/${img_base}.tar.gz"

	if [ "${package_img_tgz:-0}" != "0" ]; then

		debug "${img_base}: creating unpacked tarfile"

		( cd "${unpacked_d}" && tar \

			-S --use-compress-program=gzip-rsyncable \

			-cf "${img_base}.img.tar.gz" "${image}" ) ||

			fail "failed to create ${img_base}.img.tar.gz in ${unpacked_d}"

	fi

	# split up the image creatiom. Before this was done with part2ovf. However with the introduction

    # of armel and qemu images, the creation of disk images seperate from an OVF container is needed

	# create raw paritioned disk

	disk_img="${pub_d}/${img_base}-filesystem.img"

	if [ "${qcow2_create:-0}" -ne 0 -o "${vmdk_create:-0}" -ne 0 -o \

		 "${ovf_create:-0}" -ne 0 -o "${raw_img:-0}" -ne 0 -o \

		 "${qemu_img:-0}" -ne 0 -o "${ova_create:-0}" ]; then

	    debug "creating raw partitioned disk image"

		part2disk_opts="-v"

        [ "${nogrub:-0}" -ne 1 ] && part2disk_opts="${part2disk_opts} -G"

		part2disk="${base_d}/ovf/part2disk"

		# Certain arches need their own handling

		[[ "${arch}" =~ ppc64el ]] && part2disk="${part2disk}.${arch}"

		part2cmd="${part2disk} --size=${img_size} ${part2disk_opts} ${unpacked_d}/${image} ${disk_img}"

	    debug "  cmd: ${part2cmd}"

		$part2cmd || fail "failed to run part2cmd: ${part2cmd}"

	fi

	# create qemu images

	if [ "${qemu_img:-0}" -ne 0 ]; then

	    cmd=( "${qemu_img_cmd[@]}" )

		# %d- > base_dir, %o -> pub dir, %i -> final image, %r -> root fs

		cmd=( "${cmd[@]//%d/${base_d}}" )

		cmd=( "${cmd[@]//%r/${disk_img}}" )

		cmd=( "${cmd[@]//%i/${img_base}}" )

		cmd=( "${cmd[@]//%o/${pub_d}}" )

		cmd=( "${cmd[@]//%l/${root_fs_label}}" )

        debug "creating qemu/metal disk image with ${cmd[@]}"

        ${cmd[@]} ||

             fail "failed to run qemu_img_cmd: ${cmd[@]}"

	fi

	ovf_disks=""

	disk_out="${pub_d}/${img_base}-disk1"

	# create img files

	if [ "${qcow2_create:-0}" -ne 0 ]; then

		qcow_cmd="${base_d}/ovf/part2img --type qcow2 --source ${disk_img} --target ${disk_out} --extension img"

		debug "creating qcow2 from partitioned ${disk_img} to ${disk_out}.img"

		$qcow_cmd || fail "failed to run qcow_cmd: ${qcow_cmd}"

		[ "${qcow2_ovf:-0}" -eq 1 ] && { ovf_disks="${ovf_disks} img"; debug "qcow OVF queued for creation"; }

		[ "${ovf_default:-vmdk}" = "qcow2" ] && ovf_default="img"

	fi

	# create vmdk image

	if [ "${vmdk_create:-0}" -ne 0 ]; then

		vmdk_cmd="${base_d}/ovf/part2img --type vmdk --source ${disk_img} --target ${disk_out}"

		debug "creating vmdk from partitioned ${disk_img} to ${disk_out}.vmdk"

		$vmdk_cmd || fail "failed to run vmdk_cmd: ${vmdk_cmd}"

		[ "${vmdk_ovf:-0}" -eq 1 ] && { ovf_disks="${ovf_disks} vmdk"; debug "vmdk OVF queue for creation"; }

	fi

	# UEFI boot paths only work for {arm,amd}64

	if [[ "${arch}" =~ (amd64|arm64) ]]; then

		uefi_disk_out="${pub_d}/${img_base}-uefi1"

		# create raw UEFI/BIOS/GPT partitioned disk

		uefi_img="${pub_d}/${img_base}-uefi.img"

		if [ "${qcow2_uefi_create:-0}" -ne 0 -o ${vmdk_uefi_create:-0} -ne 0 ]; then

			debug "creating raw UEFI disk image"

			part2cmd_base="${base_d}/ovf/part2disk-gpt.${arch}"

			part2cmd="${part2cmd_base} --size=${img_size} ${part2disk_opts} ${unpacked_d}/${image} ${uefi_img}"

			debug "   cmd: ${part2cmd}"

			$part2cmd || fail "failed to run part2cmd: ${part2cmd}"

		fi

		# create UEFI qcow's

		if [ "${qcow2_uefi_create:-0}" -ne 0 ]; then

			qcow_cmd="${base_d}/ovf/part2img --type qcow2 --source ${uefi_img} --target ${uefi_disk_out} --extension img"

			debug "creating qcow2 from UEFI partitioned ${uefi_img} to ${uefi_disk_out}.img"

			$qcow_cmd || fail "failed to run qcow_cmd: ${qcow_cmd}"

			[ "${qcow2_ovf:-0}" -eq 1 ] && { ovf_disks="${ovf_disks} img"; debug "qcow OVF queued for creation"; }

			[ "${ovf_default:-vmdk}" = "qcow2" ] && ovf_default="img"

		fi

		# create UEFI vmdk's

		if [ "${vmdk_uefi_create:-0}" -ne 0 ]; then

			qcow_cmd="${base_d}/ovf/part2img --type vmdk --source ${uefi_img} --target ${uefi_disk_out}"

			debug "creating vmdk from UEFI partitioned ${uefi_img} to ${uefi_disk_out}.vmdk"

			$qcow_cmd || fail "failed to run qcow_cmd: ${qcow_cmd}"

			[ "${qcow2_ovf:-0}" -eq 1 ] && { ovf_disks="${ovf_disks} img"; debug "qcow OVF queued for creation"; }

			[ "${ovf_default:-vmdk}" = "qcow2" ] && ovf_default="img"

		fi

		[ "${raw_disk:-0}" -ne 0 ] || {	[ -e "${uefi_img}" ] && rm ${uefi_img}; }

	fi

	# GPT/PReP images for ppc64el

	if [ "${arch}" == "ppc64el" ]; then

		gpt_disk_out="${pub_d}/${img_base}-disk1"

		# create raw GPT partitioned disk

		gpt_img="${pub_d}/${img_base}-gpt.img"

		if [ "${qcow2_gpt_create:-0}" -ne 0 -o ${vmdk_gpt_create:-0} -ne 0 ]; then

			debug "creating raw GPT disk image"

			part2cmd="${base_d}/ovf/part2disk-gpt.ppc64el --size=${img_size} ${part2disk_opts} ${unpacked_d}/${image} ${gpt_img}"

			debug "   cmd: ${part2cmd}"

			$part2cmd || fail "failed to run part2cmd: ${part2cmd}"

		fi

		# create GPT qcow's

		if [ "${qcow2_gpt_create:-0}" -ne 0 ]; then

			qcow_cmd="${base_d}/ovf/part2img --type qcow2 --source ${gpt_img} --target ${gpt_disk_out} --extension img"

			debug "creating qcow2 from GPT/PReP partitioned ${gpt_img} to ${gpt_disk_out}.img"

			$qcow_cmd || fail "failed to run qcow_cmd: ${qcow_cmd}"

			[ "${qcow2_ovf:-0}" -eq 1 ] && { ovf_disks="${ovf_disks} img"; debug "qcow OVF queued for creation"; }

			[ "${ovf_default:-vmdk}" = "qcow2" ] && ovf_default="img"

		fi

		# create GPT vmdk's

		# XXX UNTESTED

		if [ "${vmdk_gpt_create:-0}" -ne 0 ]; then

			qcow_cmd="${base_d}/ovf/part2img --type vmdk --source ${gpt_img} --target ${gpt_disk_out}"

			debug "creating vmdk from GPT partitioned ${gpt_img} to ${gpt_disk_out}.vmdk"

			$qcow_cmd || fail "failed to run qcow_cmd: ${qcow_cmd}"

			[ "${qcow2_ovf:-0}" -eq 1 ] && { ovf_disks="${ovf_disks} img"; debug "qcow OVF queued for creation"; }

			[ "${ovf_default:-vmdk}" = "qcow2" ] && ovf_default="img"

		fi

		[ "${raw_disk:-0}" -ne 0 ] || {	[ -e "${gpt_img}" ] && rm ${gpt_img}; }

	fi

	# create the OVA container

	if [ "${ova_create:-0}" -eq 1 ]; then

		prefix="${img_base}"

		debug "Creating OVA for ${disk_name}"

		debug "    OVA Filename: ${ovf_d}/${prefix}.ova"

		cmd=( "${ova_cmd[@]}" )

		cmd=( "${cmd[@]//%d/${base_d}}" )

		cmd=( "${cmd[@]//%o/${ovf_d}}" )

		cmd=( "${cmd[@]//%p/${prefix}}" )

		cmd=( "${cmd[@]//%r/${disk_img}}" )

		cmd=( "${cmd[@]//%s/${img_size}}" )

		debug "creating ova with ${cmd[@]}"

		"${cmd[@]}" || 	fail "failed to run ova_cmd: ${cmd[@]}"

		sudo chown -R "$(id -u):$(id -g)" "${ovf_d}"

		chmod go+r "${ovf_d}/"*

		touch "${ovf_d}/"*

	fi

	# create OVF containers

	for disk_type in ${ovf_disks}; do

		[ "${ovf_default:-vmdk}" = "${disk_type}" ] &&

			prefix="${img_base}" ||

			prefix="${img_base}-${disk_type}"

		disk_name="${img_base}-disk1.${disk_type}"

		debug "Creating OVF for ${disk_type} disk: ${disk_name}"

		debug "    OVF Filename: ${ovf_d}/${prefix}.ovf"

		cmd=( "${ovf_cmd[@]}" )

		cmd=( "${cmd[@]//%o/${ovf_d}}" )

		cmd=( "${cmd[@]//%i/${unpacked_d}/${image}}" )

		cmd=( "${cmd[@]//%d/${base_d}}" )

		cmd=( "${cmd[@]//%p/${prefix}}" )

		cmd=( "${cmd[@]//%r/${disk_name}}" )

		cmd=( "${cmd[@]//%s/${img_size}}" )

		cmd=( "${cmd[@]//%t/${disk_type}}" )

		debug "creating ovf with ${cmd[@]}"

		"${cmd[@]}" || 	fail "failed to run ovf_cmd: ${cmd[@]}"

		sudo chown -R "$(id -u):$(id -g)" "${ovf_d}"

		chmod go+r "${ovf_d}/"*

		touch "${ovf_d}/"*

	done

	[ "${raw_disk:-0}" -ne 0 ] || {	[ -e "${disk_img}" ] && rm ${disk_img}; }

	{ [ "${copy_out_kernels}" -eq 0 ] ||

	  cp "${unpacked_d}/${kinfo}" "${pub_d}"; } &&

		cp "${unpacked_d}/${img_base}.manifest" "${pub_d}" ||

			fail "failed to copy kinfo and manifest"

	[ -e "${unpacked_d}/${img_base}.img" ] && {

			rm "${unpacked_d}/${img_base}.img" ||

				fail "failed deletion of ${img_base}.img";

		}

	if [ "${preserve_image}" = 0 ]; then

		rm -rf "${tmpdest}" || fail "failed to delete ${tmpdest}"

	fi

done

exit 0