~darkmuggle-deactivatedaccount/ubuntu-on-ec2/ec2-publishing-scripts.blacklist

#!/bin/bash
# vi: ts=4 noexpandtab

TMPD=""

error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
debug() {
	local level=${1}
	shift;
	[ "${level}" -gt "${VERBOSITY}" ] && return
	error "$(date):" "${@}"
}

Usage() {
	cat <<EOF
Usage: ${0##*/} [ options ] suite build-name release-type dir [arch]

   publish build output to ec2 from 'dir' as 'release-type'
   if arch given, only do so for 'arch' (i386 or amd64) instead of all

   suite is one of:
      karmic, lucid ...
   build-type is one of:
      server, desktop
   release type is one of:
      sandbox: 'sandbox'
      daily:   'daily'
      milestones: beta* alpha* rc*
      release: 'release'

        --serial ser      use serial rather than dirname or build-info.txt
                          for serial
   -l | --add-launch x    pass add-launch parm through to build-image
                          if not given, default is 'all'
                          use 'none' to not change perms
   -v | --verbose         increase verbosity
EOF
}


sort_published() {
	# sorts on region, arch, type
	sort --key 1,1 --key 3,3 --key 4,4 "$1"
}

# findfile(f,[dirs]): search through dirs for $f
findfile() {
	local f=${1} d="" fz=$(readlink -f "${0}");
	shift;
	[ -f "${f}" ] && { _RET=${f}; return 0; }
	for d in "${fz%/*}" "${0%/*}" "."; do
		[ -f "${d}/${f}" ] && { _RET="${d}/${f}"; return 0; }
	done
	return 1
}

cleanup() {
	[ -z "${TMPD}" -o ! -d "${TMPD}" ] || rm -Rf "${TMPD}"
}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }

# filter_list(field-number,seperator,<list>)
# filter stdinput, printing only lines the value in field-number is
# in the list <list>
filter_list() {
	local field=${1} fs=${2} reg="" i="";
	shift 2
	for i in "${@}"; do
		reg="${reg:+${reg}|}${i}"
	done
	reg="($reg)"
	awk "-F${fs}" "\$$field ~ reg { print \$0 }" "reg=${reg}"
}

short_opts="hl:v"
long_opts="add-launch:,allow-existing,help,reuse-ebs-from:,serial:,verbose"
getopt_out=$(getopt --name "${0##*/}" \
	--options "${short_opts}" --long "${long_opts}" -- "$@") &&
	eval set -- "${getopt_out}" ||
	bad_Usage

serial="";
VERBOSITY=0
label=""
add_launch_default=( --add all )
add_launch=( )
add_launch_given=0
allow_existing=""
pthrough=( )
ebs_pthrough=( )

while [ $# -ne 0 ]; do
	cur=${1}; next=${2};
	case "$cur" in
		--serial) serial=${next};;
		-h|--help) Usage; exit 0;;
		-l|--add-launch)
			add_launch_given=1
			add_launch[${#add_launch[@]}]="--add";
			add_launch[${#add_launch[@]}]="${next}";

			# special arg 'none' means "do not modify"
			[ "${next}" = "none" ] && add_launch=( );
			shift;;
		--) shift; break;;
		-v|--verbose)
			VERBOSITY=$((${VERBOSITY}+1))
			pthrough[${#pthrough[@]}]=${cur};;
		   --allow-existing)
			allow_existing="--allow-existing";;
		   --reuse-ebs-from)
			ebs_pthrough[${#ebs_pthrough[@]}]="${cur}=${next}";
			shift;;
		-*) bad_Usage "confused by ${cur}";;
	esac
	shift;
done

[ $# -lt 4 ] && bad_Usage "must provide suite, build-name, label, and dir"
[ $# -gt 5 ] && bad_Usage " too many args: $*"

suite=${1}
bd_name=${2}
label=${3}
build_d_in=${4}
arches=${5:-i386 amd64}

# for now, quick hack
case "${suite}" in
	hardy|intrepid|jaunty|karmic) publish_ebs=0;;
	*) ebs_size=8; publish_ebs=1;;
esac

publish_hvm=0
case "${suite}" in
	hardy|intrepid|jaunty|karmic|maverick) publish_hvm=0;;
	lucid)
		[ "${bd_name}" = "server" ] && publish_hvm=1
		publish_hvm=0; ## delete or comment out for lucid-server hvm builds
		;;
	*) publish_hvm=1;;
esac

which pathprefix4label >/dev/null 2>&1 ||
	fail "unable to find pathprefix4label"

pathprefix=$(pathprefix4label "${label}" image) ||
	failp "bad release type ${label}. must be one of\n\t%s\n" \
		"$(pathprefix4label --show-labels)"

bucket_prefix=${BUCKET_BASE:-ubuntu}

[ -d "${build_d_in}/unpacked" ] && build_d="${build_d_in}/unpacked" ||
	build_d=${build_d_in}

# bd_fp is full path to top level build dir (not including unpacked)
# build_d is not necissarily a full path to build_dir/unpacked
bd_fp=$(cd "${build_d}/.." && pwd)

# flabel is "filename label". 
# different from label if label is 'release', 'beta*' or 'rc*'
# in which case it has the release number

# this "when to use version in string" is copied elsewhere
# please look for VERSION2STRING if updating it.
# the idea is that beta+ get the release number in their name
rel_num=$(ubuntu-adj2version "${suite}") || fail "bad suite ${suite}"
case "${label}" in
	release)
		flabel=${rel_num};;
	beta*|rc*)
		flabel=${rel_num}-${label};;
	*) flabel=${label};;
esac

binfo="${build_d}/build-info.txt"
if [ -f "${binfo}" ]; then
	[ -n "${serial}" ] ||
		serial=$(sh -c ". '${binfo}' && echo \${serial}")
	orig_prefix=$(sh -c ". '${binfo}' && echo \${orig_prefix}")
fi

if [ -z "${serial}" ]; then
	serial=$(basename "${bd_fp}");
fi

kernel_list_in=""
case "${suite}" in
	intrepid|jaunty|karmic) kernel_list="";;
    hardy|lucid) kernel_list_in="kernels-pv-grub-hd0-V1.01.txt";;
	maverick|natty) kernel_list_in="kernels-pv-grub-hd0-V1.01.txt";;
	oneiric|precise|quantal) kernel_list_in="kernels-pv-grub-hd0-V1.02.txt";;
	*) kernel_list_in="kernels-pv-grub-hd0-V1.03.txt";;
esac

if [ -n "${kernel_list_in}" ]; then
	findfile "${kernel_list_in}" "${fullzero%/*}" ||
		fail "failed to find file ${kernel_list_in}"
	kernel_list=${_RET}
fi

TMPD=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
	fail "failed to make tempdir"
trap cleanup EXIT

export XC2_XIMAGES_CACHE_D="$TMPD/ximgcache"

all_output="${TMPD}/all-output.txt"
[ "${label}" = "daily" -o "${label}" = "sandbox" -o "${label}" = "testing" ] &&
	published_results="${build_d_in}/published-ec2-${label}.txt" ||
	published_results="${build_d_in}/published-ec2-release.txt" ||

[ ! -e "${published_results}" ] ||
	error "WARNING: overwriting ${published_results}";

all_regions=( $(ec2-list-all-regions) ) ||
	fail "failed to get list of regions"

for arch in ${arches}; do

	# find the prefix using suite-*arch.img.tar.gz
	prefix=""
	img_tgz=""

	# daily builds are named "<suite>-cloudimg-<arch>"
	# releases are named "ubuntu-<version>-cloudimg-<arch>"
	relpre="ubuntu-${rel_num}"
	for x in "${build_d}/${suite}-"*"-${arch}.manifest" \
	         "${build_d}/${relpre}"*"-${arch}.manifest" ; do
		[ -f "${x}" ] || continue
		[ -n "${prefix}" ] && fail "more than 1 manifest for ${arch}"
		prefix=${x%.manifest} && prefix=${prefix##*/}
		for f in "${bd_fp}/$prefix.tar.gz" "${build_d}/$prefix.img.tar.gz"; do
			[ -f "${f}" ] && img_tgz="${f}"
		done
	done
	[ -n "${prefix}" ] || 
		fail "could not find prefix based on *.manifest for ${build_d}"
	[ -n "${img_tgz}" ] ||
		fail "no image tarball with prefix ${prefix} in ${bd_fp} or ${build_d}"

	krd_out=${TMPD}/${arch}.krd-ids.txt
	if [ -n "${kernel_list}" ]; then
		debug 1 "using kernels from ${kernel_list_in}"
		filter_list 1 '\t' "${all_regions[@]}" \
			< "${kernel_list}" > "${krd_out}.filtered" ||
			fail "failed to filter ${kernel_list_in} for ${all_regions[*]}"

		awk '-F\t' '/^[^#]/ && $3 == arch { printf("%s\n",$0) }' \
			"arch=${arch}" 'OFS=\t' "${krd_out}.filtered" > "${krd_out}" ||
			fail "failed to read krd info from ${kernel_list}"
	else
		fail "Unable to find pvgrub list for ${region}"
	fi

	kernels=""
	initrds=""
	regions=""

	# krd output should be region id arch type name/manifest
	# we need that to be sorted by region
	sort "${krd_out}" > "${krd_out}.sort"
	while read region id xarch xtype xpath; do
		case "${xtype}" in
			ramdisk) initrds="${initrds},${id}";;
			kernel) kernels="${kernels},${id}";;
			*) failp "confused by output of publish-build-krd: %s\n" \
				 "${region} ${id} ${xarch} ${xtype} ${xpath}";;
		esac

		# if regions does not have this region, append it
		tmp=,${regions},
		[ "${tmp#*,${region},}" = "${tmp}" ] &&
			regions="${regions},${region}"
		
	done < "${krd_out}.sort"
	krd_args=( )
	[ -z "${regions}" ] ||
		krd_args=( --kernels "${kernels#,}" --ramdisks "${initrds#,}" 
		           --regions "${regions#,}" )

	cat "${krd_out}"
	cat "${krd_out}" >> "${all_output}"

	pubname="ubuntu-${suite}-${flabel}-${arch}-${bd_name}-${serial}"
	output="${TMPD}/publish-image-${arch}.txt"
	debug 1 "publishing -> ${pubname}"
	publish-image --bucket-prefix "${bucket_prefix}" \
		--resize 10G \
		--path-prefix "$pathprefix" \
		"${krd_args[@]}" --output "${output}" --type image \
		${allow_existing} \
		--rename "${pubname}" "${pthrough[@]}" "${arch}" "${img_tgz}" &&
		cat "${output}" ||
		fail "publish-image ${img##*/} for ${suite}/${bd_name}/${arch} failed"

	cat "${output}" >> "${all_output}"
done

if [ "${publish_ebs}" != "0" ]; then
	output="${TMPD}/publish-ebs.txt"
	
	pubname="${bucket_prefix}/${pathprefix}/%t/ubuntu-${suite}-${flabel}-%a-${bd_name}-${serial}"
	urlpath="${pathprefix}/ubuntu-${suite}-${flabel}-%a-${bd_name}-${serial}.tar.gz"
	hvm_args=( )
	[ "${publish_hvm}" = "1" ] && hvm_args=( --publish-hvm )

	publish-build-ebs "${pthrough[@]}" --output "${output}" \
		--published-info "${all_output}" ${allow_existing} \
		"${ebs_pthrough[@]}" \
		--snapshot-desc "%t/ubuntu-${suite}-%a-${bd_name}-${serial}" \
		--size ${ebs_size} "${hvm_args[@]}" \
		"--upload-to=${urlpath}" \
		"${build_d}" "${pubname}" &&
		cat "${output}" &&
		cat "${output}" >> "${all_output}" ||
		fail "publish-build-ebs failed."
fi

[ ${add_launch_given} -eq 0 ] && add_launch=( "${add_launch_default[@]}" )

sort_published "${all_output}" > "${published_results}" ||
	fail "failed to write results to ${published_results}"

if [ -s "${all_output}.unused" ]; then
	sort_published "${all_output}.unused" >> "${published_results}" ||
		fail "failed to write unused results to ${published_results}"
fi

if [ "${NO_WEB_INDICES:-0}" = "0" ]; then
	update-build-indexes "$label" "$bd_fp" "$suite" ||
		fail "failed to make web indices"
fi

if [ "${#add_launch[@]}" -ne 0 ]; then
	# if the final arugment of 'add_launch' is "all", then
	# call publicize-build. otherwise, just call modify-access
	if [ "${add_launch[${#add_launch[@]}-1]}" = "all" ]; then

		[ -n "${allow_existing}" ] && {
				publicize-build "${label}" "${build_d}" "${allow_existing}" 1>&2 ||
				fail "failed to publicize-build"
			}

		[ -z "${allow_existing}" ] && {
			publicize-build "${label}" "${build_d}" 1>&2 ||
				fail "failed to publicize-build"
		}
	else
		modify-access "${add_launch[@]}" "${all_output}" 1>&2 ||
			fail "failed to grant access"
	fi
fi

debug 1 "done (${SECONDS} seconds)"