~ubuntu-archive/ubuntu-archive-scripts/trunk

#! /bin/sh

set -e

LOCK=

mtimes=

cleanup () {

	[ "$mtimes" ] && rm -rf "$mtimes"

	[ "$LOCK" ] && rm -f "$LOCK"

}

trap cleanup EXIT HUP INT QUIT TERM

if [ -t 1 ]; then

	REDIR=

else

	REDIR='>/dev/null 2>&1'

fi

LOCK="$HOME/.archive-reports.lock"

if ! eval lockfile -r1 "$LOCK" $REDIR; then

	trap - EXIT HUP INT QUIT TERM

	exit 1

fi

MIRROR="$HOME/mirror/ubuntu"

GERMINATE="$HOME/mirror/ubuntu-germinate"

OUT="$HOME/public_html"

DEVEL=jammy

PROPOSED_MIGRATION_SERIES="trusty xenial bionic focal impish jammy"

mtimes="$(mktemp -d)"

collect_mtimes () {

	stat -c '%n %Y' \

		"$MIRROR"/dists/*/Release \

		"$GERMINATE/germinate.output" \

		"$HOME/extra-germinate/germinate.output" \

		>"$mtimes/$1" 2>/dev/null || true

}

mtime () {

	(grep "^$2 " "$mtimes/$1" || echo "$2 0") | cut -d' ' -f2

}

mtime_changed () {

	[ "$(mtime old "$1")" != "$(mtime new "$1")" ]

}

release_changed () {

	mtime_changed "$MIRROR/dists/$1/Release"

}

germinate_changed () {

	mtime_changed "$GERMINATE/germinate.output" || \

		mtime_changed "$HOME/extra-germinate/germinate.output"

}

is_devel () {

	case $suite in

	    $DEVEL-proposed)

		return 0

		;;

	    *)

		return 1

		;;

	esac

}

pm_pending_tests () {

	local excuses

	excuses="$HOME/public_html/proposed-migration/$1/update_excuses.html"

	[ ! -f "$excuses" ] || grep -q "Test in progress" "$excuses"

}

background () {

	local var="$1"

	shift

	"$@" &

	eval "$var=\"\${$var:+\$$var }\$!\""

}

background_wait () {

	local var="$1"

	for pid in $(eval "echo \$$var"); do

		wait "$pid"

	done

	eval "$var="

}

collect_mtimes old

mkdir -p "$MIRROR"

rsync -aq \

	--exclude dapper\* --exclude edgy\* --exclude feisty\* \

	--exclude gutsy\* --exclude hardy\* --exclude intrepid\* \

	--exclude jaunty\* --exclude karmic\* --exclude lucid\* \

	--exclude maverick\* --exclude natty\* --exclude oneiric\* \

	--exclude precise\* --exclude quantal\* --exclude raring\* \

	--exclude saucy\* --exclude utopic\* --exclude wily\* \

	--exclude yakkety\* --exclude zesty\* --exclude artful\* \

	--exclude cosmic\* --exclude disco\* --exclude eoan\* \

	--include Packages\* --include Sources\* --include Release\* \

	--include udeb.list --include \*\*/installer-\*/current \

	--include "**/$DEVEL/Contents-*" \

	--include \*/ --exclude \* --delete --delete-excluded \

	--prune-empty-dirs \

	ftpmaster.internal::ubuntu-dists/ "$MIRROR/dists/"

collect_mtimes new

# The $DEVEL-proposed chdist configuration is used by proposed-migration;

# it's important to get that done as soon as possible, so we do it first.

chdist_pids=

chdists=0

for dist in "$HOME/.chdist"/*; do

	[ -d "$dist" ] || continue

	suite="${dist##*/}"

	suite="${suite%-*}"

	if is_devel "$suite" && release_changed "$suite"; then

		background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null

		chdists=$(($chdists + 1))

	fi

	if [ "$chdists" = 7 ]; then

		background_wait chdist_pids

		chdists=0

	fi

done

background_wait chdist_pids

if mtime_changed "$MIRROR/.*/Release"; then

	background pids run-apt-mirror-snapshot

fi

pids=

run_proposed_migration () {

	DISTRIBUTION=ubuntu SERIES="$1" run-proposed-migration

}

check_git_uptodate() {

	cd "$1"

	git fetch origin -q

	branch=$(git symbolic-ref --short -q HEAD)

	[ -z "$(git diff origin/$branch..$branch)" ]

}

for series in $PROPOSED_MIGRATION_SERIES; do

	if release_changed "$series" || release_changed "$series-proposed" || \

	   release_changed "$series-updates" || \

	   pm_pending_tests "$series" || \

	   ! check_git_uptodate "$HOME/proposed-migration/data/${series}-proposed/Hints"

	then

		background pids run_proposed_migration "$series"

	fi

done

if release_changed "$DEVEL-proposed"; then

	background pids update-transitions

fi

# Now update all the other chdist configurations, and wait for that to

# finish before doing anything else.

chdist_pids=

chdists=0

for dist in "$HOME/.chdist"/*; do

	[ -d "$dist" ] || continue

	suite="${dist##*/}"

	suite="${suite%-*}"

	if ! is_devel "$suite" && release_changed "$suite"; then

		background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null

		chdists=$(($chdists + 1))

	fi

	if [ "$chdists" = 7 ]; then

		background_wait chdist_pids

		chdists=0

	fi

done

background_wait chdist_pids

if release_changed "$DEVEL"; then

	background pids cron.NBS

fi

if release_changed "$DEVEL" || release_changed "$DEVEL-proposed"; then

	# Do an extra germinate run so that we can run component-mismatches

	# against -proposed.  (We can't do this as part of LP archive

	# publishing because that would probably cause us to break our

	# 30-minute window.)

	background pids \

		env PYTHONPATH="$HOME/ubuntu-archive-tools:$HOME/germinate" \

		extra-germinate -o "$HOME/extra-germinate" ubuntu

fi

background_wait pids

rsync -aq \

	--include germinate.output \

	--exclude _\* --exclude \*.new --include "*_${DEVEL}_*" \

	--exclude \* --delete --delete-excluded \

	ftpmaster.internal::ubuntu-germinate/ "$GERMINATE/"

collect_mtimes new

if germinate_changed; then

	# Check for the marker line emitted by lp:ubuntu-archive-publishing

	# at the end of the run.  If we don't find this, then we've rsynced

	# partial output and should try again later.

	if fgrep -q 'Germination complete.' "$GERMINATE/germinate.output"; then

		component_mismatches () {

			local name="$1"

			shift

			component-mismatches \

				-d "$OUT/$name.dot" -o "$OUT/$name.txt.new" \

				--html-output-file "$OUT/$name.html" \

				--csv-file "$OUT/$name.csv" \

				--exclude "community" \

				"$@"

			if [ "$name" = component-mismatches ]; then

				process-component-mismatches-diff \

					"$OUT/$name.txt" "$OUT/$name.txt.new"

			fi

			mv "$OUT/$name.txt.new" "$OUT/$name.txt"

			dot -Tsvg -Gdpi=55 -o "$OUT/$name.svg" "$OUT/$name.dot"

		}

		background pids component_mismatches component-mismatches

		background pids component_mismatches component-mismatches-proposed \

			-s "$DEVEL-proposed" \

			--germinate-path "$HOME/extra-germinate"

		background pids priority-mismatches \

			-o "$OUT/priority-mismatches.txt" \

			--html-output-file "$OUT/priority-mismatches.html" \

			--csv-file "$OUT/priority-mismatches.csv"

		background pids architecture-mismatches \

			-o "$OUT/architecture-mismatches.txt" \

			--html-output-file "$OUT/architecture-mismatches.html" \

			--csv-file "$OUT/architecture-mismatches.csv"

		background_wait pids

	fi

fi