~ubuntu-archive/ubuntu-archive-scripts/trunk

#! /bin/sh
set -e

LOCK=
mtimes=
cleanup () {
	[ "$mtimes" ] && rm -rf "$mtimes"
	[ "$LOCK" ] && rm -f "$LOCK"
}
trap cleanup EXIT HUP INT QUIT TERM

if [ -t 1 ]; then
	REDIR=
else
	REDIR='>/dev/null 2>&1'
fi

LOCK="$HOME/.archive-reports.lock"
if ! eval lockfile -r1 "$LOCK" $REDIR; then
	trap - EXIT HUP INT QUIT TERM
	exit 1
fi

MIRROR="$HOME/mirror/ubuntu"
MIRROR_RTM="$HOME/mirror/ubuntu-rtm"
MIRROR_PHONE_OVERLAY="$HOME/mirror/phone-overlay"
GERMINATE="$HOME/mirror/ubuntu-germinate"
OUT="$HOME/public_html"
DEVEL=eoan
DEVEL_RTM=14.09
DEVEL_RTM_FACTORY=14.09-factory
PROPOSED_MIGRATION_SERIES="precise trusty xenial bionic cosmic disco eoan"
PROPOSED_MIGRATION_SERIES_RTM="14.09 14.09-factory"

mtimes="$(mktemp -d)"

collect_mtimes () {
	stat -c '%n %Y' \
		"$MIRROR"/dists/*/Release \
		"$MIRROR_RTM"/dists/*/Release \
		"$MIRROR_PHONE_OVERLAY"/dists/*/Release \
		"$GERMINATE/germinate.output" \
		"$HOME/extra-germinate/germinate.output" \
		>"$mtimes/$1" 2>/dev/null || true
}

mtime () {
	(grep "^$2 " "$mtimes/$1" || echo "$2 0") | cut -d' ' -f2
}

mtime_changed () {
	[ "$(mtime old "$1")" != "$(mtime new "$1")" ]
}

is_rtm () {
	[ -e "$MIRROR_RTM/dists/$1" ]
}

release_changed () {
	if is_rtm "$1"; then
		mtime_changed "$MIRROR_RTM/dists/$1/Release"
	else
		mtime_changed "$MIRROR/dists/$1/Release"
	fi
}

germinate_changed () {
	mtime_changed "$GERMINATE/germinate.output" || \
		mtime_changed "$HOME/extra-germinate/germinate.output"
}

is_devel () {
	case $suite in
	    $DEVEL-proposed|$DEVEL_RTM-proposed|$DEVEL_RTM_FACTORY-proposed)
		return 0
		;;
	    *)
		return 1
		;;
	esac
}

pm_pending_tests () {
	local excuses
	if is_rtm "$1"; then
		excuses="$HOME/public_html/proposed-migration/ubuntu-rtm/$1/update_excuses.html"
	else
		excuses="$HOME/public_html/proposed-migration/$1/update_excuses.html"
	fi
	grep -q "Test in progress" "$excuses"
}

background () {
	local var="$1"
	shift
	"$@" &
	eval "$var=\"\${$var:+\$$var }\$!\""
}

background_wait () {
	local var="$1"
	for pid in $(eval "echo \$$var"); do
		wait "$pid"
	done
	eval "$var="
}

collect_mtimes old

mkdir -p "$MIRROR"
rsync -aq \
	--exclude dapper\* --exclude edgy\* --exclude feisty\* \
	--exclude gutsy\* --exclude hardy\* --exclude intrepid\* \
	--exclude jaunty\* --exclude karmic\* --exclude lucid\* \
	--exclude maverick\* --exclude natty\* --exclude oneiric\* \
	--exclude quantal\* --exclude raring\* --exclude saucy\* \
	--exclude utopic\* --exclude wily\* \
	--exclude yakkety\* --exclude zesty\* --exclude artful\* \
	--include Packages\* --include Sources\* --include Release\* \
	--include udeb.list --include \*\*/installer-\*/current \
	--include "**/$DEVEL/Contents-*" \
	--include \*/ --exclude \* --delete --delete-excluded \
	--prune-empty-dirs \
	ftpmaster.internal::ubuntu-dists/ "$MIRROR/dists/"

mkdir -p "$MIRROR_RTM"
rsync -aq \
	--include Packages\* --include Sources\* --include Release\* \
	--include udeb.list --include \*\*/installer-\*/current \
	--include "**/$DEVEL_RTM/Contents-*" \
	--include "**/$DEVEL_RTM_FACTORY/Contents-*" \
	--include \*/ --exclude \* --delete --delete-excluded \
	--prune-empty-dirs \
	bilimbi.canonical.com::ubuntu-rtm-dists/ "$MIRROR_RTM/dists/"

mkdir -p "$MIRROR_PHONE_OVERLAY"
http_proxy=http://squid.internal:3128/ wget -q --mirror \
	-nH --cut-dirs=3 -np -R '*index.html*' \
	-P "$MIRROR_PHONE_OVERLAY" \
	-X '/*/*/*/*/*/*/i18n' \
	http://ppa.launchpad.net/ci-train-ppa-service/stable-phone-overlay/ubuntu/dists/

collect_mtimes new

# The $DEVEL-proposed chdist configuration is used by proposed-migration;
# it's important to get that done as soon as possible, so we do it first.
chdist_pids=
chdists=0
for dist in "$HOME/.chdist"/*; do
	[ -d "$dist" ] || continue
	suite="${dist##*/}"
	suite="${suite%-*}"
	if is_devel "$suite" && release_changed "$suite"; then
		background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null
		chdists=$(($chdists + 1))
	fi
	if [ "$chdists" = 7 ]; then
		background_wait chdist_pids
		chdists=0
	fi
done
background_wait chdist_pids

if mtime_changed "$MIRROR/.*/Release" || \
   mtime_changed "$MIRROR_RTM/.*/Release" || \
   mtime_changed "$MIRROR_PHONE_OVERLAY/.*/Release"; then
	background pids run-apt-mirror-snapshot
fi

pids=
run_proposed_migration () {
	DISTRIBUTION=ubuntu SERIES="$1" run-proposed-migration
}
for series in $PROPOSED_MIGRATION_SERIES; do
	if release_changed "$series" || release_changed "$series-proposed" || \
	   release_changed "$series-updates" || \
	   pm_pending_tests "$series"; then
		background pids run_proposed_migration "$series"
	fi
done

run_proposed_migration_rtm () {
	DISTRIBUTION=ubuntu-rtm SERIES="$1" \
		B2_CONFIG=/home/ubuntu-archive/proposed-migration/britney-rtm.conf \
		B2_CONFIG_NOBREAKALL=/home/ubuntu-archive/proposed-migration/britney_nobreakall-rtm.conf \
		run-proposed-migration
}
for series in $PROPOSED_MIGRATION_SERIES_RTM; do
	if release_changed "$series" || release_changed "$series-proposed" || \
	   release_changed "$series-updates" || \
	   pm_pending_tests "$series"; then
		background pids run_proposed_migration_rtm "$series"
	fi
done

# Both update-transitions and run-britney need this.
rsync -aq --delete --prune-empty-dirs \
	"$MIRROR/dists/" \
	"/srv/chroots/trusty-transitions/srv/transitions/mirror/ubuntu/dists/"

if release_changed "$DEVEL-proposed"; then
	background pids update-transitions
fi

if release_changed "$DEVEL"; then
	background pids run-britney
fi

# Now update all the other chdist configurations, and wait for that to
# finish before doing anything else.
chdist_pids=
chdists=0
for dist in "$HOME/.chdist"/*; do
	[ -d "$dist" ] || continue
	suite="${dist##*/}"
	suite="${suite%-*}"
	if ! is_devel "$suite" && release_changed "$suite"; then
		background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null
		chdists=$(($chdists + 1))
	fi
	if [ "$chdists" = 7 ]; then
		background_wait chdist_pids
		chdists=0
	fi
done
background_wait chdist_pids

if release_changed "$DEVEL"; then
	background pids cron.NBS
fi

nbs_rtm () {
	DISTRIBUTION=ubuntu-rtm RELEASE="$1" \
		OUTDIR="$HOME/public_html/NBS-RTM" \
		OUTFILE="$HOME/public_html/nbs-rtm.html" \
		cron.NBS
}
if release_changed "$DEVEL_RTM"; then
	background pids nbs_rtm "$DEVEL_RTM"
fi

if release_changed "$DEVEL" || release_changed "$DEVEL-proposed"; then
	# Do an extra germinate run so that we can run component-mismatches
	# against -proposed.  (We can't do this as part of LP archive
	# publishing because that would probably cause us to break our
	# 30-minute window.)
	background pids \
		env PYTHONPATH="$HOME/ubuntu-archive-tools:$HOME/germinate" \
		extra-germinate -o "$HOME/extra-germinate" ubuntu
fi

background_wait pids

rsync -aq \
	--include germinate.output \
	--exclude _\* --exclude \*.new --include "*_${DEVEL}_*" \
	--exclude \* --delete --delete-excluded \
	ftpmaster.internal::ubuntu-germinate/ "$GERMINATE/"

collect_mtimes new

if germinate_changed; then
	# Give germinate a two minute window to stop spewing to output
	# and assume it's finished, this should be more than enough time,
	# given that it outputs nearly constantly when run, and completes
	# in under three minutes, on average
	NEW_GERMINATE_MTIME="$(mtime new "$GERMINATE/germinate.output")"
	if [ $(($(date +%s) - $NEW_GERMINATE_MTIME)) -gt 120 ]; then
		component_mismatches () {
			local name="$1"
			shift
			component-mismatches \
				-d "$OUT/$name.dot" -o "$OUT/$name.txt.new" \
				--html-output-file "$OUT/$name.html" \
				--csv-file "$OUT/$name.csv" \
				"$@"
			if [ "$name" = component-mismatches ]; then
				process-component-mismatches-diff \
					"$OUT/$name.txt" "$OUT/$name.txt.new"
			fi
			mv "$OUT/$name.txt.new" "$OUT/$name.txt"
			dot -Tsvg -Gdpi=55 -o "$OUT/$name.svg" "$OUT/$name.dot"
		}
		background pids component_mismatches component-mismatches
		background pids component_mismatches component-mismatches-proposed \
			-s "$DEVEL-proposed" \
			--germinate-path "$HOME/extra-germinate"
		background pids priority-mismatches \
			-o "$OUT/priority-mismatches.txt" \
			--html-output-file "$OUT/priority-mismatches.html" \
			--csv-file "$OUT/priority-mismatches.csv"
		background pids architecture-mismatches \
			-o "$OUT/architecture-mismatches.txt" \
			--html-output-file "$OUT/architecture-mismatches.html" \
			--csv-file "$OUT/architecture-mismatches.csv"

		background_wait pids
	else
		# Artifically bump $GERMINATE/germinate.output into the past
		touch -d @$(($NEW_GERMINATE_MTIME - 1)) $GERMINATE/germinate.output
	fi
fi