~patviafore/ubuntu-on-ec2/ec2-publishing-scripts-hirsute

#!/bin/bash
# vi: ts=4 noexpandtab

export LANG=C

TEMP_D=""
error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
debug() {
	local level=${1}
	shift;
	[ ${level} -gt ${VERBOSE} ] && return
	echo "$(date): ${@}" 1>&2
}
debug_file() {
	local level=${1}; shift;
	local file=${1}; shift;
	[ ${level} -gt ${VERBOSE} ] && return
	echo "$@" 1>&2
	cat ${file} 1>&2
	echo 1>&2
}
cleanup() {
	[ -z "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}

Usage() {
	cat <<EOF
Usage: ${0##*/} [ options ] label build_dir

   Make a build "public".  This means both add 'all' access to it
   and update published data.

   label is like "daily", "alpha2", "beta1", "release"
   options:
     -d | --info-dir  dir       publish info to dir
                                [default \${PUBLISH_BASE}/query]
     --allow-existing           For query2 - replace conflicts in Q2
     --skip-modify              Skip modification of image access
EOF
}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }

ec2data_sort() {
	# ec2 files are sorted by
	#  serial (4), region (7), virtualization_type (11),
	#  then store (5) and arch (6)
	local tab="	"
	sort --uniq "--field-separator=${tab}" -k 4,4 -k 7,7 -k 11,11 -k 5,6
}

#directory tree:
#   <suite>/<build_name>/released.txt
#   <suite>/<build_name>/released.current.txt
#   <suite>/<build_name>/released-dl.txt
#   <suite>/<build_name>/released-dl.current.txt
#   <suite>/<build_name>/daily.txt
#   <suite>/<build_name>/daily.current.txt
#   <suite>/<build_name>/daily-dl.txt
#   <suite>/<build_name>/daily-dl.current.txt
#
#files contain (tab delimited):
#<suite>	<build_name>	<name>	<serial> <img-type> <arch>	<region>	<ami>	<aki>	<ari>
#karmic	server	alpha1 20100101	instance-store amd64	us-east-1	ami-c32e7f86	aki-773c6d32	ari-abcdefgh
write_info() {
	local outfile=${1} suite=${2} bname=${3} label=${4} serial=${5} imgf=${6} build_d=${7}
	debug 0 "write_info: outfile=${1} suite=${2} bname=${3} label=${4} serial=${5} imgf=${6} build_d=${7}"
	# if serial is greater or = to serial in .current, then overwrite it
	local cf_ser=0 cf_ser_fix="" serial_fix=${serial}
	url_curfile="${outfile}-dl.current.txt"
	url_outfile="${outfile}-dl.txt"
	curfile="${outfile}.current.txt"
	outfile="${outfile}.txt"
    url_file="${TEMP_D}/urls.txt"

	case ${suite} in
		precise|trusty|wily)
			write_download_query_data="true"
			;;
		*)
			write_download_query_data="false"
			;;
	esac

	local arch ec2_arches="" f="" tarball="" path="" dashed=""
	local all_arches="armel armhf i386 amd64 ppc64el arm64 s390x powerpc"
	ec2_arches=$(awk '-F\t' '{print $6}' "${outfile}" | sort -u)
	[ -n "${ec2_arches}" ] || ec2_arches="i386 amd64"
	ec2_arches=$(echo ${ec2_arches}) # carriage return to spaces

	# convert alpha3 to alpha-3, but leave 'rc' or 'alpha-3' as is.
	[ "${label#*-}" != "${label}" ] && dashed=${label} ||
		dashed=$(echo "${label}" | sed 's,\([^0-9]*\)\([0-9]\),\1-\2,');

	# this "when to use version in string" is copied elsewhere
	# please look for VERSION2STRING if updating it.
	# the idea is that beta+ get the release number in their name
	rel_num=$(ubuntu-adj2version "${suite}") || fail "bad suite ${suite}"
	case "${label}" in
		release)
			flabel=${rel_num};;
		beta*|rc*)
			flabel=${rel_num}-${label};;
		*) flabel=${label};;
	esac

	_bname="${bname%%-hwe*}"
	case "${label}" in
		testing) path="${_bname}/test_builds/${suite}/${serial}";;
		sandbox) path="${_bname}/sandbox/${suite}/${serial}";;
		release) path="${_bname}/releases/${suite}/release-${serial}";;
		rescue)  path="${_bname}/rescue_images/${suite}/${serial}";;
		daily) path="${_bname}/${suite}/${serial}";;
		*) path="${_bname}/releases/${suite}/${dashed}";;
	esac

	# for HWE publications, they are appended to the backend
	if [[ "${bname}" =~ hwe ]]; then
		path="${path}/${bname##server-}"
	fi

	local pad_ec2_arches=" ${ec2_arches} "
	for arch in ${all_arches}; do
		# pubname is the name that this build is published as in ec2
		# but also would serve as a consistent name format for users of
		# download data
		pubname="ubuntu-${suite}-${flabel}-${arch}-${bname}-${serial}"

		tarball=""
		for f in "${build_d}/"*-cloudimg-${arch}.tar.gz "${build_d}/"*-uec-${arch}.tar.gz; do
			[ -e "${f}" ] || continue
			[ -z "${tarball}" ] || {
				error "multiple matches for ${arch} build in ${build_d}";
				return 1;
			}
			[ -f "${f}" ] && tarball="${f##*/}"
		done

		if [ -z "$tarball" ]; then
			if [ "${pad_ec2_arches#* ${arch} }" != "${pad_ec2_arches}" ]; then
				# only be fatal if the arch was in the ec2 list, but not here
				error "no matches for *-cloudimg-${arch}.tar.gz in ${build_d}";
				return 1;
			else
				continue
			fi
		fi

		[ -f "${PUBLISH_BASE}/${path}/${tarball}" ] || {
			error "${path}/${tarball} does not exist in ${PUBLISH_BASE}";
			return 1;
		}

		printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \
			"${suite}" "${bname}" "${label}" "${serial}" "${arch}" \
			"${path}/${tarball}" "${pubname}" >> "${url_file}"
	done

	local ofmt="%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n"
	# cf_ser is build serial in the current file
	[ -f "${curfile}" ] &&
		cf_ser=$(awk '-F\t' 'END { print $4 }' "${curfile}")

	# if serial values do not contain a '.X', then make them '.0'
	[ "${cf_ser#.*}" = "${cf_ser}" ] &&
		cf_ser_fix=${cf_ser}.0 || cf_ser_fix=${cf_ser}
	[ "${serial#.*}" = "${serial}" ] && serial_fix=${serial}.0

	if [[ "${serial_fix}" == "${cf_ser_fix}" ||
		  "${serial_fix}" > "${cf_ser_fix}" ]]; then
		cat "${imgf}" > "${curfile}" ||
			{ error "failed to write ${curfile}"; return 1; }
		if [ ${write_download_query_data} = "true" ]; then
			cat "${url_file}" > "${url_curfile}" ||
				{ error "failed to write ${url_curfile}"; return 1; }
		fi
		error "updated ${curfile}"
	fi

	local found="" imgf_t="$TEMP_D/imgf_t"
	local cond='$1 == suite && $3 == label && $4 == serial && $2 == bname'

	if [ -e "$outfile" ]; then
		debug 0 "Outfile ${outfile} exists"
		debug_file 3 ${outfile} "Outfile contains:"
		# imgf must be complete, containing all ids for this build, or data
		# will be lost.  This strips out any data from the current build
		awk '-F\t' "(!($cond)) { print \$0; }" \
			"suite=$suite" "label=$label" "serial=$serial" "bname=$bname" \
			"$outfile" > "$imgf_t" ||
			{ error "failed to write to $imgf_t"; return 1; }
		debug_file 3 ${imgf_t} "imgf_t from outfile contains:"

		## the following would check that no records were deleted
		## but only allow updated or appended.  I'm not enabling this
		## as populating a new region might trigger it.
		#local olen="" newlen="" addlen=""
		#olen=$(wc -l < "$outfile")
		#newlen=$(wc -l < "$imgf_t")
		#addlen=$(wc -l < "$imgf")
		#[ $((${newlen}+${addlen})) -ge $olen ] ||
		#	fail "some data would be lost on update to $outfile"
	else
		: > "$imgf_t"
	fi
	cat "$imgf" >> "$imgf_t"
	debug_file 3 ${imgf_t} "imgf_t from outfile contains:"
	ec2data_sort < "$imgf_t" > "$outfile"
	error "updated ${outfile}"
	debug_file 1 ${imgf_t} "after sort outfile contains:"

	if [ ${write_download_query_data} = "true" ]; then
		# if label/serial/suite/bname exists in ${url_outfile} then do not append
		found=""
		[ -e "${url_outfile}" ] &&
			found=$(awk '-F\t' "${cond} { print \$8 }" "suite=${suite}" \
				"label=${label}" "serial=${serial}" "bname=${bname}" "${url_outfile}" )
		if [ -n "${found}" ]; then
			error "build already present in ${url_outfile}, not appending"
		else
			cat "${url_file}" >> "${url_outfile}" ||
				{ error "failed to write ${url_file} to ${url_outfile}"; return 1; }
			error "updated ${url_outfile}"
		fi
	fi
}

VERBOSE=0
short_opts="d:hv"
long_opts="help,info-dir:,skip-modify,allow-existing,verbose"
getopt_out=$(getopt --name "${0##*/}" \
	--options "${short_opts}" --long "${long_opts}" -- "$@") &&
	eval set -- "${getopt_out}" ||
	bad_Usage

allow_existing=""
skip_modify=0
info_dir=""
[ -n "${PUBLISH_BASE}" ] && info_dir="${PUBLISH_BASE}/query"

oargs=( "${0##*/}" "${@}" )
while [ $# -ne 0 ]; do
	cur=${1}; next=${2};
	case "$cur" in
		-d|--info-dir) info_dir=${next}; shift;;
	       --allow-existing) allow_existing="--allow_existing";;
           --skip-modify) skip_modify=1;;
		-h|--help) Usage; exit 0;;
		-v|--verbose) VERBOSE=$((${VERBOSE} + 1));;
		--) shift; break;;
	esac
	shift;
done

if [[ ${VERBOSE} -gt 1 ]]; then
	debug 1 "Enabling full debugging with level >1"
	set -x
fi

[ $# -eq 2 ] || bad_Usage "must provide label and build_dir"
label=${1}
build_d_in=${2}
label=${label//-/} # turn alpha-3 into alpha3

[ -n "${info_dir}" ] || bad_Usage "info-dir must be set"

[ -n "${PUBLISH_BASE}" ] || PUBLISH_BASE=$(cd "${info_dir}/.." && pwd)

build_d=$(cd "${build_d_in}" && pwd) ||
	fail "failed to cd ${build_d_in}"
build_d=${build_d%/unpacked}

build_info="${build_d}/unpacked/build-info.txt"
[ -f "${build_info}" ] || fail "unable to find build-info at ${build_info}"

serial=""
suite=""
build_name=""

. "${build_info}" || fail "failed to source build-info ${build_info}"

for v in serial suite build_name; do
	[ -n "${!v}" ] || fail "$v not set in ${build_info}"
done

out_d="${info_dir}/${suite}/${build_name}"

if [ "${label}" = "daily" -o "${label}" = "sandbox" ]; then
	published_results="${build_d}/published-ec2-${label}.txt"
	outfiles=( "${out_d}/daily" )
elif [ "${label}" = "testing" ]; then
	published_results="${build_d}/published-ec2-${label}.txt"
	outfiles=( "${out_d}/testing" )
else
	published_results="${build_d}/published-ec2-release.txt"
	outfiles=( "${out_d}/released" )
fi

[ -f "${published_results}" ] ||
	fail "results file ${published_results} does not exist"

{ [ -d "${out_d}" ] || mkdir -p "${out_d}"; } ||
	fail "failed to make dir ${out_d}"

if [ ! -d "${info_dir}/.bzr" ]; then
	( cd "${info_dir}" && bzr init && bzr add --quiet . &&
	  bzr commit --quiet -m "initial state" ) >/dev/null &&
		error "initialized bzr directory in ${info_dir}" ||
		fail "failed to initialize bzr directory in ${info_dir}"
fi

debug_file 0 ${published_results} "Published results"

if [ $skip_modify -eq 0 ]; then
	debug 0 "Modifying access to AMIs"
	modify-access --add all "${published_results}" ||
		fail "failed to grant access"
else
	debug 0 "Instructed to keep these images private"
fi

TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX")
trap cleanup EXIT

# create a set of region.<region>.list files so we can
# run a single describe-images per region
while read region ami arch img_type manifest rest; do
	debug 0 $region $ami $arch $img_type $manifest $rest
	[ "${ami}" != "${ami#ami-}" ] || continue
	debug 0 "Including ${ami} in ${region} published results"
	echo "${ami}" >> "${TEMP_D}/region.${region}.list" ||
		fail "failed to write ${ami} to ${TEMP_D}/region.${region}.list"
done < "${published_results}"

for rfile in "${TEMP_D}/"region.*.list; do
	region=${rfile##*/region.}; region=${region%.list}
	amis=$(cat "${rfile}")
	imgf="${TEMP_D}/images.${region}.txt"
    debug 0 "Querying region ${region} image AMIs: ${amis}"
    try=5
    while (( ${try} > 0 )) ; do
    	xc2 describe-images --region "${region}" $amis > "${imgf}" ||
    		fail "failed to describe in ${region}: ${amis}"
        found_all=1
        for AMI in ${amis}; do
            grep ${AMI} ${imgf} || { found_all=0 && break; }
        done
        if (( ${found_all} == 1 )) ; then
            debug 0 "Found all expected published AMIs"
            break
        else
            error "Missing ${AMI} in ${region}"
            try=$((${try} - 1))
            if (( ${try} > 0 )); then
                error "Sleeping 5 minutes"
                sleep 300
                error "Retry #" $((5 - ${try}))
            else
                fail "Unable to find AMI after 25 minutes."
            fi
        fi
    done
	debug_file 0 ${imgf} "Images in region ${region}"
	# in describe-images output:
	#    $2 = ami-id
	#   $10 = aki-id
	#   $11 = ari-id
	awk '-F\t' '$1 == "IMAGE" {
			arch=$8; if ($8 == "x86_64") arch = "amd64";
			itype=$13; if (itype == "") itype="instance-store"
			vtype=$14; if (vtype == "") vtype="paravirtual"
            name=$3; disk="";
            if (name ~ "io1") disk="-io1";
            if (name ~ "ssd") disk="-ssd";
            printf("%s\t%s\t%s\t%s\t%s%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
                suite,bname,label,ser,itype,disk,arch,reg,$2,$10,$11,vtype)
		}' "suite=$suite" "bname=$build_name" "label=${label}" \
		   "ser=${serial}" "reg=${region}" "${imgf}" >> "${TEMP_D}/images.txt"
done

debug_file 0 ${TEMP_D}/images.txt "Images unsorted:"
ec2data_sort < "$TEMP_D/images.txt" > "$TEMP_D/images.txt.sorted"
debug_file 0 ${TEMP_D}/images.txt.sorted "Images sorted:"

for outf in "${outfiles[@]}"; do
	write_info "${outf}" "${suite}" "${build_name}" "${label}" \
		"${serial}" "${TEMP_D}/images.txt.sorted" "${build_d}" ||
		fail "failed to write to ${outf}"
done

# if any of the current.txt files (most likely the one just written)
# is newer than the latest file, the re-write the latest file
for r in daily released; do
	newer=""
 	latest="${info_dir}/${r}.latest.txt"
	for f in "${info_dir}"/*/*/${r}-dl.current.txt; do
		[ -f "${f}" ] && [ ! -f "${latest}" -o "$f" -nt "${latest}" ] &&
			{ newer=${f}; break; }
	done
	[ -n "$newer" ] || continue
	# the latest files contain only the first 4 fields
	awk '-F\t' 'FNR == 1 && NR != 1 { print prev; }
	     { prev = sprintf("%s\t%s\t%s\t%s",$1,$2,$3,$4); }
	     END { print prev ; } ' \
	     "${info_dir}"/*/*/${r}-dl.current.txt > "${latest}" ||
		fail "failed to update ${latest}"
	error "updated ${latest}"
done

bzr add "${info_dir}"
if bzr diff "${info_dir}" >/dev/null; then
	error "no changes were made to ${info_dir}"
else
	error "committing changes for ${info_dir}"
	bzr commit -m "${oargs[*]}" "${info_dir}" ||
		fail "failed to bzr commit in ${info_dir}"
fi

if [ -f "${info_dir}/daily.latest.txt" ]; then
	error "Updating daily manifest"
	write-site-manifest "${info_dir}/daily.latest.txt" "${PUBLISH_BASE}" \
		> "${PUBLISH_BASE}/.manifest-daily"
	debug_file 0 ${PUBLISH_BASE}/.manifest-daily "Daily manifest:"
else
	error "Emptying daily manifest"
	# empty the file, there was no daily.latest, so empty manifest-daily
	: > "${PUBLISH_BASE}/.manifest-daily"
fi

if [ -f "${info_dir}/released.latest.txt" ]; then
	error "Updating release manifest"
	write-site-manifest "${info_dir}/released.latest.txt" "${PUBLISH_BASE}" \
		> "${PUBLISH_BASE}/.manifest"
	debug_file 0 ${PUBLISH_BASE}/.manifest-daily "Release manifest:"
else
	error "Emptying release manifest"
	# empty the file, there was no released.latest, so empty .manifest
	: > "${PUBLISH_BASE}/.manifest"
fi

exit 0