~ubuntu-archive/ubuntu-archive-scripts/trunk

#! /bin/sh
set -e

LOCK=
mtimes=
cleanup () {
	[ "$mtimes" ] && rm -rf "$mtimes"
	[ "$LOCK" ] && rm -f "$LOCK"
}
trap cleanup EXIT HUP INT QUIT TERM

if [ -t 1 ]; then
	REDIR=
else
	REDIR='>/dev/null 2>&1'
fi

LOCK="$HOME/.archive-reports.lock"
if ! eval lockfile -r1 "$LOCK" $REDIR; then
	trap - EXIT HUP INT QUIT TERM
	exit 1
fi

MIRROR="$HOME/mirror/ubuntu"
GERMINATE="$HOME/mirror/ubuntu-germinate"
OUT="$HOME/public_html"
DEVEL=groovy
PROPOSED_MIGRATION_SERIES="precise trusty xenial bionic focal groovy"

mtimes="$(mktemp -d)"

collect_mtimes () {
	stat -c '%n %Y' \
		"$MIRROR"/dists/*/Release \
		"$GERMINATE/germinate.output" \
		"$HOME/extra-germinate/germinate.output" \
		>"$mtimes/$1" 2>/dev/null || true
}

mtime () {
	(grep "^$2 " "$mtimes/$1" || echo "$2 0") | cut -d' ' -f2
}

mtime_changed () {
	[ "$(mtime old "$1")" != "$(mtime new "$1")" ]
}

release_changed () {
	mtime_changed "$MIRROR/dists/$1/Release"
}

germinate_changed () {
	mtime_changed "$GERMINATE/germinate.output" || \
		mtime_changed "$HOME/extra-germinate/germinate.output"
}

is_devel () {
	case $suite in
	    $DEVEL-proposed)
		return 0
		;;
	    *)
		return 1
		;;
	esac
}

pm_pending_tests () {
	local excuses
	excuses="$HOME/public_html/proposed-migration/$1/update_excuses.html"
	[ ! -f "$excuses" ] || grep -q "Test in progress" "$excuses"
}

background () {
	local var="$1"
	shift
	"$@" &
	eval "$var=\"\${$var:+\$$var }\$!\""
}

background_wait () {
	local var="$1"
	for pid in $(eval "echo \$$var"); do
		wait "$pid"
	done
	eval "$var="
}

collect_mtimes old

mkdir -p "$MIRROR"
rsync -aq \
	--exclude dapper\* --exclude edgy\* --exclude feisty\* \
	--exclude gutsy\* --exclude hardy\* --exclude intrepid\* \
	--exclude jaunty\* --exclude karmic\* --exclude lucid\* \
	--exclude maverick\* --exclude natty\* --exclude oneiric\* \
	--exclude quantal\* --exclude raring\* --exclude saucy\* \
	--exclude utopic\* --exclude wily\* \
	--exclude yakkety\* --exclude zesty\* --exclude artful\* \
	--exclude cosmic\* \
	--include Packages\* --include Sources\* --include Release\* \
	--include udeb.list --include \*\*/installer-\*/current \
	--include "**/$DEVEL/Contents-*" \
	--include \*/ --exclude \* --delete --delete-excluded \
	--prune-empty-dirs \
	ftpmaster.internal::ubuntu-dists/ "$MIRROR/dists/"

collect_mtimes new

# The $DEVEL-proposed chdist configuration is used by proposed-migration;
# it's important to get that done as soon as possible, so we do it first.
chdist_pids=
chdists=0
for dist in "$HOME/.chdist"/*; do
	[ -d "$dist" ] || continue
	suite="${dist##*/}"
	suite="${suite%-*}"
	if is_devel "$suite" && release_changed "$suite"; then
		background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null
		chdists=$(($chdists + 1))
	fi
	if [ "$chdists" = 7 ]; then
		background_wait chdist_pids
		chdists=0
	fi
done
background_wait chdist_pids

if mtime_changed "$MIRROR/.*/Release"; then
	background pids run-apt-mirror-snapshot
fi

pids=
run_proposed_migration () {
	DISTRIBUTION=ubuntu SERIES="$1" run-proposed-migration
}
for series in $PROPOSED_MIGRATION_SERIES; do
	if release_changed "$series" || release_changed "$series-proposed" || \
	   release_changed "$series-updates" || \
	   pm_pending_tests "$series" || \
	   ! bzr missing -d "$HOME/proposed-migration/data/${series}-proposed/Hints"
	then
		background pids run_proposed_migration "$series"
	fi
done

# Both update-transitions and run-britney need this.
rsync -aq --delete --prune-empty-dirs \
	"$MIRROR/dists/" \
	"/srv/chroots/trusty-transitions/srv/transitions/mirror/ubuntu/dists/"

if release_changed "$DEVEL-proposed"; then
	background pids update-transitions
fi

if release_changed "$DEVEL"; then
	background pids run-britney
fi

# Now update all the other chdist configurations, and wait for that to
# finish before doing anything else.
chdist_pids=
chdists=0
for dist in "$HOME/.chdist"/*; do
	[ -d "$dist" ] || continue
	suite="${dist##*/}"
	suite="${suite%-*}"
	if ! is_devel "$suite" && release_changed "$suite"; then
		background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null
		chdists=$(($chdists + 1))
	fi
	if [ "$chdists" = 7 ]; then
		background_wait chdist_pids
		chdists=0
	fi
done
background_wait chdist_pids

if release_changed "$DEVEL"; then
	background pids cron.NBS
fi

if release_changed "$DEVEL" || release_changed "$DEVEL-proposed"; then
	# Do an extra germinate run so that we can run component-mismatches
	# against -proposed.  (We can't do this as part of LP archive
	# publishing because that would probably cause us to break our
	# 30-minute window.)
	background pids \
		env PYTHONPATH="$HOME/ubuntu-archive-tools:$HOME/germinate" \
		extra-germinate -o "$HOME/extra-germinate" ubuntu
fi

background_wait pids

rsync -aq \
	--include germinate.output \
	--exclude _\* --exclude \*.new --include "*_${DEVEL}_*" \
	--exclude \* --delete --delete-excluded \
	ftpmaster.internal::ubuntu-germinate/ "$GERMINATE/"

collect_mtimes new

if germinate_changed; then
	# Check for the marker line emitted by lp:ubuntu-archive-publishing
	# at the end of the run.  If we don't find this, then we've rsynced
	# partial output and should try again later.
	if fgrep -q 'Germination complete.' "$GERMINATE/germinate.output"; then
		component_mismatches () {
			local name="$1"
			shift
			component-mismatches \
				-d "$OUT/$name.dot" -o "$OUT/$name.txt.new" \
				--html-output-file "$OUT/$name.html" \
				--csv-file "$OUT/$name.csv" \
				--exclude "community" \
				"$@"
			if [ "$name" = component-mismatches ]; then
				process-component-mismatches-diff \
					"$OUT/$name.txt" "$OUT/$name.txt.new"
			fi
			mv "$OUT/$name.txt.new" "$OUT/$name.txt"
			dot -Tsvg -Gdpi=55 -o "$OUT/$name.svg" "$OUT/$name.dot"
		}
		background pids component_mismatches component-mismatches
		background pids component_mismatches component-mismatches-proposed \
			-s "$DEVEL-proposed" \
			--germinate-path "$HOME/extra-germinate"
		background pids priority-mismatches \
			-o "$OUT/priority-mismatches.txt" \
			--html-output-file "$OUT/priority-mismatches.html" \
			--csv-file "$OUT/priority-mismatches.csv"
		background pids architecture-mismatches \
			-o "$OUT/architecture-mismatches.txt" \
			--html-output-file "$OUT/architecture-mismatches.html" \
			--csv-file "$OUT/architecture-mismatches.csv"

		background_wait pids
	fi
fi