1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
#! /bin/sh
set -e
LOCK=
mtimes=
cleanup () {
[ "$mtimes" ] && rm -rf "$mtimes"
[ "$LOCK" ] && rm -f "$LOCK"
}
trap cleanup EXIT HUP INT QUIT TERM
if [ -t 1 ]; then
REDIR=
else
REDIR='>/dev/null 2>&1'
fi
LOCK="$HOME/.archive-reports.lock"
if ! eval lockfile -r1 "$LOCK" $REDIR; then
trap - EXIT HUP INT QUIT TERM
exit 1
fi
MIRROR="$HOME/mirror/ubuntu"
GERMINATE="$HOME/mirror/ubuntu-germinate"
OUT="$HOME/public_html"
DEVEL=trusty
mtimes="$(mktemp -d)"
collect_mtimes () {
stat -c '%n %Y' \
"$MIRROR"/dists/*/Release \
"$GERMINATE/germinate.output" \
"$HOME/extra-germinate/germinate.output" \
>"$mtimes/$1" 2>/dev/null
}
mtime () {
(grep "^$2 " "$mtimes/$1" || echo "$2 0") | cut -d' ' -f2
}
mtime_changed () {
[ "$(mtime old "$1")" != "$(mtime new "$1")" ]
}
release_changed () {
mtime_changed "$MIRROR/dists/$1/Release"
}
germinate_changed () {
mtime_changed "$GERMINATE/germinate.output" || \
mtime_changed "$HOME/extra-germinate/germinate.output"
}
background () {
local var="$1"
shift
"$@" &
eval "$var=\"\${$var:+\$$var }\$!\""
}
background_wait () {
local var="$1"
for pid in $(eval "echo \$$var"); do
wait "$pid"
done
eval "$var="
}
collect_mtimes old
rsync -aq \
--exclude dapper\* --exclude edgy\* --exclude feisty\* \
--exclude gutsy\* --exclude hardy\* --exclude intrepid\* \
--exclude jaunty\* --exclude karmic\* --exclude maverick\* \
--exclude natty\* --exclude oneiric\* \
--include Packages\* --include Sources\* --include Release\* \
--include udeb.list --include \*\*/installer-\*/current \
--include "**/$DEVEL/Contents-*" \
--include \*/ --exclude \* --delete --prune-empty-dirs \
ftpmaster.internal::ubuntu-dists/ "$MIRROR/dists/"
collect_mtimes new
# The $DEVEL-proposed chdist configuration is used by proposed-migration;
# it's important to get that done as soon as possible, so we do it first.
chdist_pids=
chdists=0
for dist in "$HOME/.chdist"/*; do
[ -d "$dist" ] || continue
suite="${dist##*/}"
suite="${suite%-*}"
if [ "$suite" = "$DEVEL-proposed" ] && release_changed "$suite"; then
background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null
chdists=$(($chdists + 1))
fi
if [ "$chdists" = 5 ]; then
background_wait chdist_pids
chdists=0
fi
done
background_wait chdist_pids
pids=
if release_changed "$DEVEL" || release_changed "$DEVEL-proposed"; then
background pids run-proposed-migration
fi
if release_changed "$DEVEL-proposed"; then
rsync -aq --delete --prune-empty-dirs \
"$MIRROR/dists/$DEVEL/" \
"/srv/chroots/lucid-transitions/srv/transitions/mirror/ubuntu/dists/$DEVEL/"
rsync -aq --delete --prune-empty-dirs \
"$MIRROR/dists/$DEVEL-proposed/" \
"/srv/chroots/lucid-transitions/srv/transitions/mirror/ubuntu/dists/$DEVEL-proposed/"
background pids update-transitions
fi
# Now update all the other chdist configurations, and wait for that to
# finish before doing anything else.
chdist_pids=
chdists=0
for dist in "$HOME/.chdist"/*; do
[ -d "$dist" ] || continue
suite="${dist##*/}"
suite="${suite%-*}"
if [ "$suite" != "$DEVEL-proposed" ] && release_changed "$suite"; then
background chdist_pids chdist apt-get "${dist##*/}" update >/dev/null
chdists=$(($chdists + 1))
fi
if [ "$chdists" = 5 ]; then
background_wait chdist_pids
chdists=0
fi
done
background_wait chdist_pids
if release_changed "$DEVEL"; then
background pids run-britney
background pids cron.NBS
fi
if release_changed "$DEVEL" || release_changed "$DEVEL-proposed"; then
# Do an extra germinate run so that we can run component-mismatches
# against -proposed. (We can't do this as part of LP archive
# publishing because that would probably cause us to break our
# 30-minute window.)
background pids \
env PYTHONPATH="$HOME/ubuntu-archive-tools:$HOME/germinate" \
extra-germinate -o "$HOME/extra-germinate" ubuntu
fi
background_wait pids
rsync -aq \
--include germinate.output \
--exclude _\* --include "*_${DEVEL}_*" \
--exclude \* --delete \
ftpmaster.internal::ubuntu-germinate/ "$GERMINATE/"
collect_mtimes new
if germinate_changed; then
# Give germinate a two minute window to stop spewing to output
# and assume it's finished, this should be more than enough time,
# given that it outputs nearly constantly when run, and completes
# in under three minutes, on average
NEW_GERMINATE_MTIME="$(mtime new "$GERMINATE/germinate.output")"
if [ $(($(date +%s) - $NEW_GERMINATE_MTIME)) -gt 120 ]; then
component_mismatches () {
local name="$1"
shift
component-mismatches \
-d "$OUT/$name.dot" -o "$OUT/$name.txt.new" \
"$@"
if [ "$name" = component-mismatches ]; then
process-component-mismatches-diff \
"$OUT/$name.txt" "$OUT/$name.txt.new"
fi
mv "$OUT/$name.txt.new" "$OUT/$name.txt"
dot -Tsvg -Gdpi=55 -o "$OUT/$name.svg" "$OUT/$name.dot"
}
background pids component_mismatches component-mismatches
background pids component_mismatches component-mismatches-proposed \
-s "$DEVEL-proposed" \
--germinate-path "$HOME/extra-germinate"
background pids priority-mismatches -o "$OUT/priority-mismatches.txt"
background pids architecture-mismatches -o "$OUT/architecture-mismatches.txt"
background_wait pids
else
# Artifically bump $GERMINATE/germinate.output into the past
touch -d @$(($NEW_GERMINATE_MTIME - 1)) $GERMINATE/germinate.output
fi
fi
|