~ubuntu-on-ec2/ubuntu-on-ec2/ec2-publishing-scripts.hvm-ephemeral

#!/bin/bash
# vi: ts=4 noexpandtab

export LANG=C

TEMP_D=""
error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
debug() {
	local level=${1}
	shift;
	[ "${level}" -gt "${VERBOSITY}" ] && return
	error "$(date):" "${@}"
}
cleanup() {
	[ -z "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}

Usage() {
	cat <<EOF
Usage: ${0##*/} [ options ] label build_dir

   Make a build "public".  This means both add 'all' access to it
   and update published data.

   label is like "daily", "alpha2", "beta1", "release"
   options:
     -d | --info-dir  dir       publish info to dir 
                                [default \${PUBLISH_BASE}/query]
EOF
}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }

ec2data_sort() {
	# ec2 files are sorted by 
	#  serial (4), region (7), virtualization_type (11), 
	#  then store (5) and arch (6)
	local tab="	"
	sort --uniq "--field-separator=${tab}" -k 4,4 -k 7,7 -k 11,11 -k 5,6
}

#directory tree:
#   <suite>/<build_name>/released.txt
#   <suite>/<build_name>/released.current.txt
#   <suite>/<build_name>/released-dl.txt
#   <suite>/<build_name>/released-dl.current.txt
#   <suite>/<build_name>/daily.txt
#   <suite>/<build_name>/daily.current.txt
#   <suite>/<build_name>/daily-dl.txt
#   <suite>/<build_name>/daily-dl.current.txt
#
#files contain (tab delimited):
#<suite>	<build_name>	<name>	<serial> <img-type> <arch>	<region>	<ami>	<aki>	<ari>
#karmic	server	alpha1 20100101	instance-store amd64	us-east-1	ami-c32e7f86	aki-773c6d32	ari-abcdefgh
write_info() {
	local outfile=${1} suite=${2} bname=${3} label=${4} serial=${5} imgf=${6} build_d=${7}
	# if serial is greater or = to serial in .current, then overwrite it
	local cf_ser=0 cf_ser_fix="" serial_fix=${serial}
	url_curfile="${outfile}-dl.current.txt"
	url_outfile="${outfile}-dl.txt"
	curfile="${outfile}.current.txt"
	outfile="${outfile}.txt"
    url_file="${TEMP_D}/urls.txt"

	local arch arches="i386 amd64 armel" f="" tarball="" path="" dashed=""
	arches=$(awk '-F\t' '{print $6}' "${outfile}" | sort -u)
	[ -n "${arches}" ] || arches="i386 amd64"

	# convert alpha3 to alpha-3, but leave 'rc' or 'alpha-3' as is.
	[ "${label#*-}" != "${label}" ] && dashed=${label} ||
		dashed=$(echo "${label}" | sed 's,\([^0-9]*\)\([0-9]\),\1-\2,');

	# this "when to use version in string" is copied elsewhere
	# please look for VERSION2STRING if updating it.
	# the idea is that beta+ get the release number in their name
	rel_num=$(ubuntu-adj2version "${suite}") || fail "bad suite ${suite}"
	case "${label}" in
		release)
			flabel=${rel_num};;
		beta*|rc*)
			flabel=${rel_num}-${label};;
		*) flabel=${label};;
	esac

	case "${label}" in
		release) path="${bname}/releases/${suite}/release-${serial}";;
		daily) path="${bname}/${suite}/${serial}";;
		*) path="${bname}/releases/${suite}/${dashed}";;
	esac

	for arch in ${arches}; do
		# pubname is the name that this build is published as in ec2
		# but also would serve as a consistent name format for users of
		# download data
		pubname="ubuntu-${suite}-${flabel}-${arch}-${bname}-${serial}"

		tarball=""
		for f in "${build_d}/"*-cloudimg-${arch}.tar.gz "${build_d}/"*-uec-${arch}.tar.gz; do
			[ -e "${f}" ] || continue
			[ -z "${tarball}" ] || {
				error "multiple matches for ${arch} build in ${build_d}";
				return 1;
			}
			[ -f "${f}" ] && tarball="${f##*/}"
		done
		[ -n "${tarball}" ] ||
			{ error "no matches for *-cloudimg-${arch}.tar.gz in ${build_d}"; return 1; }
		
		[ -f "${PUBLISH_BASE}/${path}/${tarball}" ] || {
			error "${path}/${tarball} does not exist in ${PUBLISH_BASE}";
			return 1;
		}

		printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \
			"${suite}" "${bname}" "${label}" "${serial}" "${arch}" \
			"${path}/${tarball}" "${pubname}" >> "${url_file}"
	done

	local ofmt="%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n"
	# cf_ser is build serial in the current file
	[ -f "${curfile}" ] &&
		cf_ser=$(awk '-F\t' 'END { print $4 }' "${curfile}")

	# if serial values do not contain a '.X', then make them '.0'
	[ "${cf_ser#.*}" = "${cf_ser}" ] && 
		cf_ser_fix=${cf_ser}.0 || cf_ser_fix=${cf_ser}
	[ "${serial#.*}" = "${serial}" ] && serial_fix=${serial}.0

	if [[ "${serial_fix}" == "${cf_ser_fix}" || 
		  "${serial_fix}" > "${cf_ser_fix}" ]]; then
		cat "${imgf}" > "${curfile}" ||
			{ error "failed to write ${curfile}"; return 1; }
		cat "${url_file}" > "${url_curfile}" ||
			{ error "failed to write ${url_curfile}"; return 1; }
		error "updated ${curfile}"
	fi

	local found="" imgf_t="$TEMP_D/imgf_t"
	local cond='$1 == suite && $3 == label && $4 == serial && $2 == bname'
    
	if [ -e "$outfile" ]; then
		# imgf must be complete, containing all ids for this build, or data
		# will be lost.  This strips out any data from the current build
		awk '-F\t' "(!($cond)) { print \$0; }" \
			"suite=$suite" "label=$label" "serial=$serial" "bname=$bname" \
			"$outfile" > "$imgf_t" ||
			{ error "failed to write to $imgf_t"; return 1; }

		## the following would check that no records were deleted
		## but only allow updated or appended.  I'm not enabling this
		## as populating a new region might trigger it.
		#local olen="" newlen="" addlen=""
		#olen=$(wc -l < "$outfile")
		#newlen=$(wc -l < "$imgf_t")
		#addlen=$(wc -l < "$imgf")
		#[ $((${newlen}+${addlen})) -ge $olen ] ||
		#	fail "some data would be lost on update to $outfile"
	else
		: > "$imgf_t"
	fi
	cat "$imgf" >> "$imgf_t"
	ec2data_sort < "$imgf_t" > "$outfile"
	error "updated ${outfile}"

	# if label/serial/suite/bname exists in ${url_outfile} then do not append
	found=""
	[ -e "${url_outfile}" ] &&
		found=$(awk '-F\t' "${cond} { print \$8 }" "suite=${suite}" \
			"label=${label}" "serial=${serial}" "bname=${bname}" "${url_outfile}" )
	if [ -n "${found}" ]; then
		error "build already present in ${url_outfile}, not appending"
	else
 		cat "${url_file}" >> "${url_outfile}" ||
			{ error "failed to write ${url_file} to ${url_outfile}"; return 1; }
		error "updated ${url_outfile}"
	fi
}

short_opts="d:h"
long_opts="help,info-dir:,skip-modify"
getopt_out=$(getopt --name "${0##*/}" \
	--options "${short_opts}" --long "${long_opts}" -- "$@") &&
	eval set -- "${getopt_out}" ||
	bad_Usage

skip_modify=0
info_dir=""
[ -n "${PUBLISH_BASE}" ] && info_dir="${PUBLISH_BASE}/query"

oargs=( "${0##*/}" "${@}" )
while [ $# -ne 0 ]; do
	cur=${1}; next=${2};
	case "$cur" in
		-d|--info-dir) info_dir=${next}; shift;;
           --skip-modify) skip_modify=1;;
		-h|--help) Usage; exit 0;;
		--) shift; break;;
	esac
	shift;
done

[ $# -eq 2 ] || bad_Usage "must provide label and build_dir"
label=${1}
build_d_in=${2}
label=${label//-/} # turn alpha-3 into alpha3

[ -n "${info_dir}" ] || bad_Usage "info-dir must be set"

[ -n "${PUBLISH_BASE}" ] || PUBLISH_BASE=$(cd "${info_dir}/.." && pwd)

build_d=$(cd "${build_d_in}" && pwd) ||
	fail "failed to cd ${build_d_in}"
build_d=${build_d%/unpacked}

build_info="${build_d}/unpacked/build-info.txt"
[ -f "${build_info}" ] || fail "unable to find build-info at ${build_info}"


serial=""
suite=""
build_name=""

. "${build_info}" || fail "failed to source build-info ${build_info}"

for v in serial suite build_name; do
	[ -n "${!v}" ] || fail "$v not set in ${build_info}"
done

out_d="${info_dir}/${suite}/${build_name}"

if [ "${label}" = "daily" -o "${label}" = "sandbox" ]; then
	published_results="${build_d}/published-ec2-${label}.txt"
	outfiles=( "${out_d}/daily" )
else
	published_results="${build_d}/published-ec2-release.txt"
	outfiles=( "${out_d}/released" )
fi

[ -f "${published_results}" ] ||
	fail "results file ${published_results} does not exist"

{ [ -d "${out_d}" ] || mkdir -p "${out_d}"; } ||
	fail "failed to make dir ${out_d}"

if [ ! -d "${info_dir}/.bzr" ]; then
	( cd "${info_dir}" && bzr init && bzr add --quiet . &&
	  bzr commit --quiet -m "initial state" ) >/dev/null && 
		error "initialized bzr directory in ${info_dir}" ||
		fail "failed to initialize bzr directory in ${info_dir}"
fi


if [ $skip_modify -eq 0 ]; then
	modify-access --add all "${published_results}" ||
		fail "failed to grant access"
fi

TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX")
trap cleanup EXIT

export XC2_XIMAGES_CACHE_D=${XC2_XIMAGES_CACHE_D:-"$TEMP_D/ximgcache"}

# create a set of region.<region>.list files so we can
# run a single describe-images per region
while read region ami arch img_type manifest rest; do
	[ "${ami}" != "${ami#ami-}" ] || continue
	echo "${ami}" >> "${TEMP_D}/region.${region}.list" ||
		fail "failed to write ${ami} to ${TEMP_D}/region.${region}.list"
done < "${published_results}"

for rfile in "${TEMP_D}/"region.*.list; do
	region=${rfile##*/region.}; region=${region%.list}
	amis=$(cat "${rfile}")
	imgf="${TEMP_D}/images.${region}.txt"
	xc2 ximages describe-images --region "${region}" $amis > "${imgf}" ||
		fail "failed to describe in ${region}: ${amis}"
	# in describe-images output:
	#    $2 = ami-id
	#   $10 = aki-id
	#   $11 = ari-id
	awk '-F\t' '$1 == "IMAGE" {
			arch=$8; if ($8 == "x86_64") arch = "amd64";
			itype=$13; if (itype == "") itype="instance-store"
			vtype=$14; if (vtype == "") vtype="paravirtual"
			printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
				suite,bname,label,ser,itype,arch,reg,$2,$10,$11,vtype)
		}' "suite=$suite" "bname=$build_name" "label=${label}" \
		   "ser=${serial}" "reg=${region}" "${imgf}" >> "${TEMP_D}/images.txt"
done

ec2data_sort < "$TEMP_D/images.txt" > "$TEMP_D/images.txt.sorted"

for outf in "${outfiles[@]}"; do
	write_info "${outf}" "${suite}" "${build_name}" "${label}" \
		"${serial}" "${TEMP_D}/images.txt.sorted" "${build_d}" ||
		fail "failed to write to ${outf}"
done

# if any of the current.txt files (most likely the one just written)
# is newer than the latest file, the re-write the latest file
for r in daily released; do 
	newer=""
 	latest="${info_dir}/${r}.latest.txt"
	for f in "${info_dir}"/*/*/${r}.current.txt; do
		[ -f "${f}" ] && [ ! -f "${latest}" -o "$f" -nt "${latest}" ] &&
			{ newer=${f}; break; }
	done
	[ -n "$newer" ] || continue
	# the latest files contain only the first 4 fields
	awk '-F\t' 'FNR == 1 && NR != 1 { print prev; } 
	     { prev = sprintf("%s\t%s\t%s\t%s",$1,$2,$3,$4); }
	     END { print prev ; } ' \
	     "${info_dir}"/*/*/${r}.current.txt > "${latest}" ||
		fail "failed to update ${latest}"
	error "updated ${latest}"
done

bzr add "${info_dir}"
if bzr diff "${info_dir}" >/dev/null; then
	error "no changes were made to ${info_dir}"
else
	bzr commit -m "${oargs[*]}" "${info_dir}" ||
		fail "failed to bzr commit in ${info_dir}"
fi

if [ -f "${info_dir}/daily.latest.txt" ]; then
	write-site-manifest "${info_dir}/daily.latest.txt" "${PUBLISH_BASE}" \
		> "${PUBLISH_BASE}/.manifest-daily"
else
	# empty the file, there was no daily.latest, so empty manifest-daily
	: > "${PUBLISH_BASE}/.manifest-daily"
fi

if [ -f "${info_dir}/released.latest.txt" ]; then
	write-site-manifest "${info_dir}/released.latest.txt" "${PUBLISH_BASE}" \
		> "${PUBLISH_BASE}/.manifest"
else
	# empty the file, there was no released.latest, so empty .manifest
	: > "${PUBLISH_BASE}/.manifest"
fi

exit 0