~patviafore/ubuntu-on-ec2/ec2-publishing-scripts-hirsute

#!/bin/bash -x
# vi: ts=4 noexpandtab

TMPD=""

dist_ge() { [[ "$1" > "$2" || "$1" == "$2" ]]; }
error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
debug() {
	local level=${1}
	shift;
	[ "${level}" -gt "${VERBOSITY}" ] && return
	echo "$(date): ${@}" 1>&2
}

Usage() {
	cat <<EOF
Usage: ${0##*/} [ options ] suite build-name release-type dir [arch]

   publish build output to ec2 from 'dir' as 'release-type'
   if arch given, only do so for 'arch' (i386 or amd64) instead of all

   suite is one of:
      karmic, lucid ...
   build-type is one of:
      server, desktop
   release type is one of:
      sandbox: 'sandbox'
      daily:   'daily'
      milestones: beta* alpha* rc*
      release: 'release'

        --serial ser      use serial rather than dirname or build-info.txt
                          for serial
   -l | --add-launch x    pass add-launch parm through to build-image
                          if not given, default is 'all'
                          use 'none' to not change perms
   -v | --verbose         increase verbosity
EOF
}


sort_published() {
	# sorts on region, arch, type
	sort --key 1,1 --key 3,3 --key 4,4 "$1"
}

# findfile(f,[dirs]): search through dirs for $f
findfile() {
	local f=${1} d="" fz=$(readlink -f "${0}");
	shift;
	[ -f "${f}" ] && { _RET=${f}; return 0; }
	for d in "${fz%/*}" "${0%/*}" "."; do
		[ -f "${d}/${f}" ] && { _RET="${d}/${f}"; return 0; }
	done
	return 1
}

cleanup() {
	[ -z "${TMPD}" -o ! -d "${TMPD}" ] || rm -Rf "${TMPD}"
}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }

# filter_list(field-number,seperator,<list>)
# filter stdinput, printing only lines the value in field-number is
# in the list <list>
filter_list() {
	local field=${1} fs=${2} reg="" i="";
	shift 2
	for i in "${@}"; do
		reg="${reg:+${reg}|}${i}"
	done
	reg="($reg)"
	awk "-F${fs}" "\$$field ~ reg { print \$0 }" "reg=${reg}"
}

find_kernel_list() {
	# Amazon has asked that we use the lastest PVGRUB
	# in all builds.

	# arg1: should be hd0 or hd00
	kernel_list_in="kernels-pv-grub-${1}.txt"
	if [ -n "${kernel_list_in}" ]; then
		findfile "${kernel_list_in}" "${fullzero%/*}" ||
			fail "failed to find file ${kernel_list_in}"
		kernel_list=${_RET}
	fi
	echo ${kernel_list}
}


short_opts="hl:v"
long_opts="add-launch:,allow-existing,help,reuse-ebs-from:,serial:,verbose"
getopt_out=$(getopt --name "${0##*/}" \
	--options "${short_opts}" --long "${long_opts}" -- "$@") &&
	eval set -- "${getopt_out}" ||
	bad_Usage

serial="";
VERBOSITY=0
label=""
add_launch_default=( --add all )
add_launch=( )
add_launch_given=0
allow_existing=""
pthrough=( )
ebs_pthrough=( )

while [ $# -ne 0 ]; do
	cur=${1}; next=${2};
	case "$cur" in
		--serial) serial=${next};;
		-h|--help) Usage; exit 0;;
		-l|--add-launch)
			add_launch_given=1
			add_launch[${#add_launch[@]}]="--add";
			add_launch[${#add_launch[@]}]="${next}";

			# special arg 'none' means "do not modify"
			[ "${next}" = "none" ] && add_launch=( );
			shift;;
		--) shift; break;;
		-v|--verbose)
			VERBOSITY=$((${VERBOSITY}+1))
			pthrough[${#pthrough[@]}]=${cur};;
		   --allow-existing)
			allow_existing="--allow-existing";;
		   --reuse-ebs-from)
			ebs_pthrough[${#ebs_pthrough[@]}]="${cur}=${next}";
			shift;;
		-*) bad_Usage "confused by ${cur}";;
	esac
	shift;
done

[ $# -lt 4 ] && bad_Usage "must provide suite, build-name, label, and dir"
[ $# -gt 5 ] && bad_Usage " too many args: $*"

suite=${1}
bd_name=${2}
label=${3}
build_d_in=${4}
arches=${5:-i386 amd64}

publish_uefi=1      # use uefi images for HVM
use_sriov_hvm=1     # Register with sriov simple
publish_i386=1      # Publish i386 images
disk_img_suffix="-disk1.img"
case "${suite}" in
	precise)
		publish_uefi=0
		use_sriov_hvm=0;;
	trusty)
		;;
	wily|xenial)
		# For Vivid and later, we only publish 64-bit AMIs.
		publish_uefi=0
		publish_i386=0;;
	yakkety|*)
		publish_i386=0
		# We stopped having separate UEFI images in yakkety; this will use the
		# "disk1" images
		publish_uefi=0
		disk_img_suffix=".img"
		;;
esac

[ "${publish_i386:-0}" -eq 0 ] && arches="${5:-amd64}"
[ "${use_sriov_hvm:-0}" -eq 1 ] && sriov_arg="--sriov"

# Just output some stuff
cat << EOM
Publishing with the following settings

suite=${suite}
bd_name=${bd_name}
label=${label}
build_d_in=${build_d_in}
use_sriov_hvm=${use_sriov_hvm}
publish_uefi=${publish_uefi}
arches=${arches}
EOM

which pathprefix4label >/dev/null 2>&1 ||
	fail "unable to find pathprefix4label"

pathprefix=$(pathprefix4label "${label}" image) ||
	failp "bad release type ${label}. must be one of\n\t%s\n" \
		"$(pathprefix4label --show-labels)"

bucket_prefix=${BUCKET_BASE:-ubuntu}

[ -d "${build_d_in}/unpacked" ] && build_d="${build_d_in}/unpacked" ||
	build_d=${build_d_in}

# bd_fp is full path to top level build dir (not including unpacked)
# build_d is not necissarily a full path to build_dir/unpacked
bd_fp=$(cd "${build_d}/.." && pwd)

# flabel is "filename label".
# different from label if label is 'release', 'beta*' or 'rc*'
# in which case it has the release number

# this "when to use version in string" is copied elsewhere
# please look for VERSION2STRING if updating it.
# the idea is that beta+ get the release number in their name
rel_num=$(ubuntu-adj2version "${suite}") || fail "bad suite ${suite}"
case "${label}" in
	release)
		flabel=${rel_num};;
	beta*|rc*)
		flabel=${rel_num}-${label};;
	*) flabel=${label};;
esac

binfo="${build_d}/build-info.txt"
if [ -f "${binfo}" ]; then
	[ -n "${serial}" ] ||
		serial=$(sh -c ". '${binfo}' && echo \${serial}")
	orig_prefix=$(sh -c ". '${binfo}' && echo \${orig_prefix}")
fi

if [ -z "${serial}" ]; then
	serial=$(basename "${bd_fp}");
fi

TMPD=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
	fail "failed to make tempdir"
trap cleanup EXIT

export XC2_XIMAGES_CACHE_D="$TMPD/ximgcache"

all_output="${TMPD}/all-output.txt"
[ "${label}" = "daily" -o "${label}" = "sandbox" -o "${label}" = "testing" ] &&
	published_results="${build_d_in}/published-ec2-${label}.txt" ||
	published_results="${build_d_in}/published-ec2-release.txt" ||

[ ! -e "${published_results}" ] ||
	error "WARNING: overwriting ${published_results}";

all_regions=( $(ec2-list-all-regions) ) ||
	fail "failed to get list of regions"

# Export environment variables used by called scripts publish-hvm-image and publish-build-ebs
export SERIAL=${serial}
export SUITE=${suite}
export RELEASE_TYPE=${flabel}
export RELEASE_LABEL=${label}

for arch in ${arches}; do

	# find the prefix using suite-*arch.img.tar.gz
	prefix=""
    img_f=""

	# daily builds are named "<suite>-cloudimg-<arch>"
	# releases are named "ubuntu-<version>-cloudimg-<arch>"
	relpre="ubuntu-${rel_num}"
    for x in "${build_d}/${suite}-"*"-${arch}.manifest" \
             "${build_d}/${relpre}"*"-${arch}.manifest" ; do
        [ -f "${x}" ] || continue
        [ -n "${prefix}" ] && fail "more than 1 manifest for ${arch}"
        prefix=${x%.manifest} && prefix=${prefix##*/}

		# Look for matching files
        candidate_fns=("${bd_fp}/$prefix.tar.gz"
		              "${build_d}/$prefix.img.tar.gz")

        for f in "${candidate_fns[@]}"; do
            [ -f "${f}" ] && img_f="${f}"
        done
    done
	[ -n "${prefix}" ] ||
		fail "could not find prefix based on *.manifest for ${build_d}"
	[ -n "${img_f}" ] ||
		fail "no image with prefix ${prefix} in ${bd_fp} or ${build_d}"

	krd_out=${TMPD}/${arch}.krd-ids.txt

	# Get the right PVGRUB based on the circumstances
	pv_grub_type="hd0"
	kernel_list=$(find_kernel_list ${pv_grub_type})

	if [ -n "${kernel_list}" ]; then
		debug 1 "using kernels from ${kernel_list_in}"
		filter_list 1 '\t' "${all_regions[@]}" \
			< "${kernel_list}" > "${krd_out}.filtered" ||
			fail "failed to filter ${kernel_list_in} for ${all_regions[*]}"

		awk '-F\t' '/^[^#]/ && $3 == arch { printf("%s\n",$0) }' \
			"arch=${arch}" 'OFS=\t' "${krd_out}.filtered" > "${krd_out}" ||
			fail "failed to read krd info from ${kernel_list}"
	else
		fail "Unable to find pvgrub list for ${region}"
	fi

	kernels=""
	initrds=""
	regions=""

	# krd output should be region id arch type name/manifest
	# we need that to be sorted by region
	sort "${krd_out}" > "${krd_out}.sort"
	while read region id xarch xtype xpath; do
		case "${xtype}" in
			ramdisk) initrds="${initrds},${id}";;
			kernel) kernels="${kernels},${id}";;
			*) failp "confused by output of find_kernel_list: %s\n" \
				 "${region} ${id} ${xarch} ${xtype} ${xpath}";;
		esac

		# if regions does not have this region, append it
		tmp=,${regions},
		[ "${tmp#*,${region},}" = "${tmp}" ] &&
			regions="${regions},${region}"

	done < "${krd_out}.sort"
	krd_args=( )
	[ -z "${regions}" ] ||
		krd_args=( --kernels "${kernels#,}" --ramdisks "${initrds#,}"
		           --regions "${regions#,}" )

	cat "${krd_out}"
	cat "${krd_out}" >> "${all_output}"

	pubname="ubuntu-${suite}-${flabel}-${arch}-${bd_name}-${serial}"
	output="${TMPD}/publish-image-${arch}.txt"
	debug 1 "publishing -> ${pubname}"

	publish-image --bucket-prefix "${bucket_prefix}" \
		--resize 10G \
		--path-prefix "$pathprefix" \
		"${krd_args[@]}" --output "${output}" --type image \
		${allow_existing} \
		--rename "${pubname}" "${pthrough[@]}" "${arch}" "${img_f}" &&
		cat "${output}" ||
		fail "publish-image ${img##*/} for ${suite}/${bd_name}/${arch} failed"

	cat "${output}" >> "${all_output}"

	# Publish the image as HVM
	if [ "${arch}" == "amd64" ]; then
		# Use $prefix-disk1.img if not using the UEFI images
		if [ "${publish_uefi}" -eq 1 ]; then
		    img_hvm="${build_d_in}/$prefix-uefi1.img"
		else
			img_hvm="${build_d_in}/$prefix$disk_img_suffix"
		fi

		debug 1 "using ${img_hvm} for HVM instance store"
		[ -e "${img_f}" ] ||
			fail "unable to find ${img_f}"

		export LOCAL_IMG_HVM_FILE=${img_hvm}

		# Publish HVM instance store AMI's
		pubname="ubuntu-${suite}-${flabel}-${arch}-${bd_name}-${serial}"
		output="${TMPD}/publish-hvm-image-${arch}.txt"
		debug 1 "publishing hvm -> ${pubname}"
		publish-hvm-image \
			--bucket-prefix "${bucket_prefix}" \
			--resize 10G \
			--path-prefix "${pathprefix}/hvm-instance" \
			--output "${output}" \
			${allow_existing} ${sriov_arg} \
			--rename "${pubname}" "${pthrough[@]}" "${arch}" "${img_hvm}" &&
			cat "${output}" ||
			fail "publish-hvm-image ${img##*/} for ${suite}/${bd_name}/${arch} failed"

		cat "${output}" >> "${all_output}"
    fi
done

# Publish ebs
output="${TMPD}/publish-ebs.txt"
pubname="${bucket_prefix}/${pathprefix}/%t/ubuntu-${suite}-${flabel}-%a-${bd_name}-${serial}"
urlpath="${pathprefix}/ubuntu-${suite}-${flabel}-%a-${bd_name}-${serial}.tar.gz"

ebs_size=8
[ "${publish_i386:-0}" -eq 0 ] && export EBS_SKIP_ARCH="i386"
publish-build-ebs "${pthrough[@]}" --output "${output}" \
	--published-info "${all_output}" ${allow_existing} \
	"${ebs_pthrough[@]}" \
	--snapshot-desc "%t/ubuntu-${suite}-%a-${bd_name}-${serial}" \
	--size ${ebs_size} --publish-hvm ${sriov_arg} \
	"--upload-to=${urlpath}" \
	"${build_d}" "${pubname}" &&
	cat "${output}" &&
	cat "${output}" >> "${all_output}" ||
	fail "publish-build-ebs failed."

[ ${add_launch_given} -eq 0 ] && add_launch=( "${add_launch_default[@]}" )

sort_published "${all_output}" > "${published_results}" ||
	fail "failed to write results to ${published_results}"

if [ -s "${all_output}.unused" ]; then
	sort_published "${all_output}.unused" >> "${published_results}" ||
		fail "failed to write unused results to ${published_results}"
fi

if [ "${NO_WEB_INDICES:-0}" = "0" ]; then
	update-build-indexes "$label" "$bd_fp" "$suite" ||
		fail "failed to make web indices"
fi

if [ "${#add_launch[@]}" -ne 0 ]; then
	# if the final arugment of 'add_launch' is "all", then
	# call publicize-build. otherwise, just call modify-access
	if [ "${add_launch[${#add_launch[@]}-1]}" = "all" ]; then

		[ -n "${allow_existing}" ] && {
				publicize-build "${pthrough[@]}" "${label}" "${build_d}" "${allow_existing}" 1>&2 ||
				fail "failed to publicize-build"
			}

		[ -z "${allow_existing}" ] && {
			publicize-build "${pthrough[@]}" "${label}" "${build_d}" 1>&2 ||
				fail "failed to publicize-build"
		}
	else
		modify-access "${add_launch[@]}" "${all_output}" 1>&2 ||
			fail "failed to grant access"
	fi
fi

debug 1 "done (${SECONDS} seconds)"