~patviafore/ubuntu-on-ec2/ec2-publishing-scripts-hirsute

#!/bin/bash

OWNER_ID="099720109477"

export LC_ALL=C

Usage() {

	cat <<EOF

Usage: ${0##*/} [outdir]

  outdir defaults to ./outd

  update a /query tree based data in ec2

  Note, this would be lossy for past images (dailies that have fallen off)

  this  makes an attempt to save old data. you could populate the

  output dir with:

      rsync cloud-images.ubuntu.com::uec-images/query/ outd -az --stats

EOF

}

error() { echo "$@" 1>&2; }

fail() { [ $# -eq 0 ] || error "$@"; exit 1; }

cleanup() {

	[ -z "${TMPD}" -o ! -d "${TMPD}" ] || rm -Rf "${TMPD}"

}

ec2data_sort() {

	# ec2 files are sorted by 

	#  serial (4), region (7), virtualization_type (11), 

	#  then store (5) and arch (6)

	local tab=$'\t'

	sort --uniq "--field-separator=${tab}" -k 4,4 -k 7,7 -k 11,11 -k 5,6

}

printdata() {

	local bdir=${1} region=$2

	local loc=$region oifs=$IFS

	[ "$region" = "us-east-1" ] && loc="us" ||

		{ [ "$region" = "eu-west-1" ] && loc="eu"; }

	local bp="ubuntu[-/]images"

	local mdif="" manif iarch itype ami aki ari vt

	while IFS="|" read manif iarch itype ami aki ari vt; do

		mdir="${manif#${OWNER_ID}/}"

		case "$mdir" in

			ubuntu/*) mdir=${mdir#ubuntu/}; mdir="ubuntu/${mdir%%/*}";;

			*) mdir=${mdir%%/*};;

		esac

		mbname=${manif##*/}

		mbname=${mbname%.manifest.xml}

		case "${mdir}" in

			${bp}-testing-$loc|${bp}-testing|${bp}-testing-dev) rline=testing;;

			${bp}-sandbox-$loc|${bp}-sandbox) continue;;

			${bp}-$loc|${bp}-milestone-$loc|$bp|$bp-milestone) rline=release;;

			*) fail "unknown manifest/name: $manif ($mdir)";

		esac

		[ "$iarch" = "x86_64" ] && arch=amd64 || arch=i386

		IFS="-"; set -- ${mbname}; IFS=${oifs}

		suite=${2}

		local label=""

		# spin (build name) is always the second to last token

		if [ $# -eq 7 ]; then

			serial=$7

 			spin=$6

			label=$4

		elif [ $# -eq 6 ]; then

			serial=$6

			spin=$5

			case "$3" in

				[0-9]*) label="release";;

				*) label=$3;;

			esac

		else

			fail "confused by name ${mbname}"

		fi

		error "label=$label spin=$spin mbname=$mbname"

		dir="${bdir}/${suite}/${spin}"

		print=( printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n"

			"$suite" "$spin" "$label" "$serial"

			"$itype" "$arch" "$region" "$ami" "$aki" "$ari" "$vt" )

		[ -d "${dir}" ] || mkdir -p "${dir}" || return 1

		if [ "${label}" = "daily" ]; then

			"${print[@]}" >> "${dir}/daily.txt"

		else

			"${print[@]}" >> "${dir}/released.txt"

		fi

		[ $? -eq 0 ] || return 1

	done

}

TEMP_D=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX)

trap cleanup EXIT

export XC2_XIMAGES_CACHE_D=${XC2_XIMAGES_CACHE_D:-"$TEMP_D/ximgcache"}

[ $# -eq 0 ] && { Usage 1>&2; exit 1; }

[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }

outd="$1"

[ -d "$outd" ] || mkdir -p "$outd"

regions=( $(ec2-list-all-regions) ) || fail "failed to get list of regions"

# save off the originals

for f in ${outd}/*/*/*; do

	[ -f "$f" ] || continue

	case "$f" in

		*/released.txt|*/daily.txt) :;;

		*) continue;;

	esac

	cp "$f" "$f.orig" || fail "failed to save $f to $f.orig"

done

name_match="^(${OWNER_ID}/|^)ubuntu[/-]images"

for r in ${regions[@]}; do

	f="${TEMP_D}/images.$r.txt"

	error "updating image cache for $r"

	xc2 ximages describe-images --region "${r}" --all  > "${f}" ||

		fail "failed to describe-images in region $r"

	ufile="$TEMP_D/ubuntu.images.$r.txt"

	awk '-F\t' '$1 == "IMAGE" && $3 ~ ubum && $6 == "public" {

		print($3,$8,$13,$2,$10,$11,$14); }' \

		"ubum=$name_match" OFS="|" "$f" > "$ufile" ||

		fail "failed to get ubuntu images from $ufile"

		printdata "${outd}" "$r" < "$ufile" ||

		fail "failed to print ubuntu images in $r"

done

# now update the .current files from the released.txt

for f in ${outd}/*/*/*; do

	case "$f" in

		*/released.txt|*/daily.txt) :;;

		*) continue;;

	esac

	# append to the file any original data.

	if [ -f "$f.orig" ]; then

		 cat "$f.orig" >> "$f" && rm "$f.orig" ||

			fail "failed to update $f with orig data"

	fi

	# sort results

	ec2data_sort < "$f" > "$f.new"

	mv "${f}.new" "${f}"

	error "wrote to $f"

	cur="${f%.txt}.current.txt"

	# get the bottom entry, and populate current with stuff like it

	tail -n 1 "$f" > "$TEMP_D/latest" &&

		read suite bname label serial other < "$TEMP_D/latest" ||

		fail "failed to read final entry in $f"

	cond='$1 == suite && $2 == bname && $3 == label && $4 == serial'

	awk '-F\t' "$cond { print \$0 }" \

		"suite=$suite" "bname=$bname" "label=$label" "serial=$serial" \

		"$f" > "$cur" ||

		fail "failed to update current $cur"

	error "wrote to $cur"

done

# this goes through each of the suite/buildname/daily.current.txt 

# files and then writes the first 4 fields of each of those files into

# the current.txt file at the top level.

for r in daily released; do

	tail -qn 1 */*/"$r.current.txt" > "$TEMP_D/$r.current" &&

		awk '-F\t' '{print $1,$2,$3,$4}' 'OFS=\t' \

			"$TEMP_D/$r.current" > "$outd/$r.latest.txt" ||

		fail "failed to write to $outd/$r.latest.txt"

	error "updated $r.latest.txt"

done

# vi: ts=4 noexpandtab