← Back to branch summary

~ubuntu-archive/ubuntu-archive-scripts/trunk

« back to all changes in this revision

Viewing changes to archive-reports

  • Committer: Steve Langasek
  • Date: 2023-12-19 06:16:57 UTC
  • Revision ID: steve.langasek@canonical.com-20231219061657-3vtta1sc7bxqirn5
Handle the rdepends/ALL subtree with rsync instead of diff.

This is a symlink tree and thus immune to patching.  germinate-output has
been out of date for some time as a result of non-zero exit codes from diff,
this should fix that.

Show diffs side-by-side

added added

removed removed

Lines of Context:
archive-reports
22
22
fi
23
23
 
24
24
MIRROR="$HOME/mirror/ubuntu"
25
 
MIRROR_RTM="$HOME/mirror/ubuntu-rtm"
26
 
MIRROR_PHONE_OVERLAY="$HOME/mirror/phone-overlay"
27
25
GERMINATE="$HOME/mirror/ubuntu-germinate"
28
26
OUT="$HOME/public_html"
29
 
DEVEL=eoan
30
 
DEVEL_RTM=14.09
31
 
DEVEL_RTM_FACTORY=14.09-factory
32
 
PROPOSED_MIGRATION_SERIES="precise trusty xenial bionic cosmic disco eoan"
33
 
PROPOSED_MIGRATION_SERIES_RTM="14.09 14.09-factory"
 
27
DEVEL=noble
 
28
PROPOSED_MIGRATION_SERIES="trusty xenial bionic focal jammy lunar mantic noble"
34
29
 
35
30
mtimes="$(mktemp -d)"
36
31
 
37
32
collect_mtimes () {
38
33
        stat -c '%n %Y' \
39
34
                "$MIRROR"/dists/*/Release \
40
 
                "$MIRROR_RTM"/dists/*/Release \
41
 
                "$MIRROR_PHONE_OVERLAY"/dists/*/Release \
42
35
                "$GERMINATE/germinate.output" \
43
36
                "$HOME/extra-germinate/germinate.output" \
44
37
                >"$mtimes/$1" 2>/dev/null || true
52
45
        [ "$(mtime old "$1")" != "$(mtime new "$1")" ]
53
46
}
54
47
 
55
 
is_rtm () {
56
 
        [ -e "$MIRROR_RTM/dists/$1" ]
57
 
}
58
 
 
59
48
release_changed () {
60
 
        if is_rtm "$1"; then
61
 
                mtime_changed "$MIRROR_RTM/dists/$1/Release"
62
 
        else
63
 
                mtime_changed "$MIRROR/dists/$1/Release"
64
 
        fi
 
49
        mtime_changed "$MIRROR/dists/$1/Release"
65
50
}
66
51
 
67
52
germinate_changed () {
71
56
 
72
57
is_devel () {
73
58
        case $suite in
74
 
            $DEVEL-proposed|$DEVEL_RTM-proposed|$DEVEL_RTM_FACTORY-proposed)
 
59
            $DEVEL-proposed)
75
60
                return 0
76
61
                ;;
77
62
            *)
82
67
 
83
68
pm_pending_tests () {
84
69
        local excuses
85
 
        if is_rtm "$1"; then
86
 
                excuses="$HOME/public_html/proposed-migration/ubuntu-rtm/$1/update_excuses.html"
87
 
        else
88
 
                excuses="$HOME/public_html/proposed-migration/$1/update_excuses.html"
89
 
        fi
90
 
        grep -q "Test in progress" "$excuses"
 
70
        excuses="$HOME/public_html/proposed-migration/$1/update_excuses.html"
 
71
        [ ! -f "$excuses" ] || grep -q "Test in progress" "$excuses"
91
72
}
92
73
 
93
74
background () {
113
94
        --exclude gutsy\* --exclude hardy\* --exclude intrepid\* \
114
95
        --exclude jaunty\* --exclude karmic\* --exclude lucid\* \
115
96
        --exclude maverick\* --exclude natty\* --exclude oneiric\* \
116
 
        --exclude quantal\* --exclude raring\* --exclude saucy\* \
117
 
        --exclude utopic\* --exclude wily\* \
 
97
        --exclude precise\* --exclude quantal\* --exclude raring\* \
 
98
        --exclude saucy\* --exclude utopic\* --exclude wily\* \
118
99
        --exclude yakkety\* --exclude zesty\* --exclude artful\* \
 
100
        --exclude cosmic\* --exclude disco\* --exclude eoan\* \
 
101
        --exclude groovy\* --exclude hirsute\* --exclude impish\* \
 
102
        --exclude kinetic\* \
119
103
        --include Packages\* --include Sources\* --include Release\* \
120
104
        --include udeb.list --include \*\*/installer-\*/current \
121
 
        --include "**/$DEVEL/Contents-*" \
 
105
        --include Contents-\* \
122
106
        --include \*/ --exclude \* --delete --delete-excluded \
123
107
        --prune-empty-dirs \
124
108
        ftpmaster.internal::ubuntu-dists/ "$MIRROR/dists/"
125
109
 
126
 
mkdir -p "$MIRROR_RTM"
127
 
rsync -aq \
128
 
        --include Packages\* --include Sources\* --include Release\* \
129
 
        --include udeb.list --include \*\*/installer-\*/current \
130
 
        --include "**/$DEVEL_RTM/Contents-*" \
131
 
        --include "**/$DEVEL_RTM_FACTORY/Contents-*" \
132
 
        --include \*/ --exclude \* --delete --delete-excluded \
133
 
        --prune-empty-dirs \
134
 
        bilimbi.canonical.com::ubuntu-rtm-dists/ "$MIRROR_RTM/dists/"
135
 
 
136
 
mkdir -p "$MIRROR_PHONE_OVERLAY"
137
 
http_proxy=http://squid.internal:3128/ wget -q --mirror \
138
 
        -nH --cut-dirs=3 -np -R '*index.html*' \
139
 
        -P "$MIRROR_PHONE_OVERLAY" \
140
 
        -X '/*/*/*/*/*/*/i18n' \
141
 
        http://ppa.launchpad.net/ci-train-ppa-service/stable-phone-overlay/ubuntu/dists/
142
 
 
143
110
collect_mtimes new
144
111
 
145
112
# The $DEVEL-proposed chdist configuration is used by proposed-migration;
161
128
done
162
129
background_wait chdist_pids
163
130
 
164
 
if mtime_changed "$MIRROR/.*/Release" || \
165
 
   mtime_changed "$MIRROR_RTM/.*/Release" || \
166
 
   mtime_changed "$MIRROR_PHONE_OVERLAY/.*/Release"; then
167
 
        background pids run-apt-mirror-snapshot
168
 
fi
169
 
 
170
131
pids=
171
132
run_proposed_migration () {
172
133
        DISTRIBUTION=ubuntu SERIES="$1" run-proposed-migration
173
134
}
 
135
 
 
136
check_git_uptodate() {
 
137
        cd "$1"
 
138
        git fetch origin -q
 
139
        branch=$(git symbolic-ref --short -q HEAD)
 
140
        [ -z "$(git diff origin/$branch..$branch)" ]
 
141
}
 
142
 
174
143
for series in $PROPOSED_MIGRATION_SERIES; do
175
144
        if release_changed "$series" || release_changed "$series-proposed" || \
176
145
           release_changed "$series-updates" || \
177
 
           pm_pending_tests "$series"; then
 
146
           pm_pending_tests "$series" || \
 
147
           ! check_git_uptodate "$HOME/proposed-migration/data/${series}-proposed/Hints"
 
148
        then
178
149
                background pids run_proposed_migration "$series"
179
150
        fi
180
151
done
181
152
 
182
 
run_proposed_migration_rtm () {
183
 
        DISTRIBUTION=ubuntu-rtm SERIES="$1" \
184
 
                B2_CONFIG=/home/ubuntu-archive/proposed-migration/britney-rtm.conf \
185
 
                B2_CONFIG_NOBREAKALL=/home/ubuntu-archive/proposed-migration/britney_nobreakall-rtm.conf \
186
 
                run-proposed-migration
187
 
}
188
 
for series in $PROPOSED_MIGRATION_SERIES_RTM; do
189
 
        if release_changed "$series" || release_changed "$series-proposed" || \
190
 
           release_changed "$series-updates" || \
191
 
           pm_pending_tests "$series"; then
192
 
                background pids run_proposed_migration_rtm "$series"
193
 
        fi
194
 
done
195
 
 
196
 
# Both update-transitions and run-britney need this.
197
 
rsync -aq --delete --prune-empty-dirs \
198
 
        "$MIRROR/dists/" \
199
 
        "/srv/chroots/trusty-transitions/srv/transitions/mirror/ubuntu/dists/"
200
 
 
201
153
if release_changed "$DEVEL-proposed"; then
202
154
        background pids update-transitions
203
155
fi
204
156
 
205
 
if release_changed "$DEVEL"; then
206
 
        background pids run-britney
207
 
fi
208
 
 
209
157
# Now update all the other chdist configurations, and wait for that to
210
158
# finish before doing anything else.
211
159
chdist_pids=
229
177
        background pids cron.NBS
230
178
fi
231
179
 
232
 
nbs_rtm () {
233
 
        DISTRIBUTION=ubuntu-rtm RELEASE="$1" \
234
 
                OUTDIR="$HOME/public_html/NBS-RTM" \
235
 
                OUTFILE="$HOME/public_html/nbs-rtm.html" \
236
 
                cron.NBS
237
 
}
238
 
if release_changed "$DEVEL_RTM"; then
239
 
        background pids nbs_rtm "$DEVEL_RTM"
240
 
fi
241
 
 
242
180
if release_changed "$DEVEL" || release_changed "$DEVEL-proposed"; then
243
181
        # Do an extra germinate run so that we can run component-mismatches
244
182
        # against -proposed.  (We can't do this as part of LP archive
245
183
        # publishing because that would probably cause us to break our
246
184
        # 30-minute window.)
247
185
        background pids \
248
 
                env PYTHONPATH="$HOME/ubuntu-archive-tools:$HOME/germinate" \
 
186
                env PYTHONPATH="${PYTHONPATH:+$PYTHONPATH:}$HOME/ubuntu-archive-tools:$HOME/germinate" \
249
187
                extra-germinate -o "$HOME/extra-germinate" ubuntu
250
188
fi
251
189
 
260
198
collect_mtimes new
261
199
 
262
200
if germinate_changed; then
263
 
        # Give germinate a two minute window to stop spewing to output
264
 
        # and assume it's finished, this should be more than enough time,
265
 
        # given that it outputs nearly constantly when run, and completes
266
 
        # in under three minutes, on average
267
 
        NEW_GERMINATE_MTIME="$(mtime new "$GERMINATE/germinate.output")"
268
 
        if [ $(($(date +%s) - $NEW_GERMINATE_MTIME)) -gt 120 ]; then
 
201
        # Check for the marker line emitted by lp:ubuntu-archive-publishing
 
202
        # at the end of the run.  If we don't find this, then we've rsynced
 
203
        # partial output and should try again later.
 
204
        if fgrep -q 'Germination complete.' "$GERMINATE/germinate.output"; then
269
205
                component_mismatches () {
270
206
                        local name="$1"
271
207
                        shift
273
209
                                -d "$OUT/$name.dot" -o "$OUT/$name.txt.new" \
274
210
                                --html-output-file "$OUT/$name.html" \
275
211
                                --csv-file "$OUT/$name.csv" \
 
212
                                --exclude "community-maas" \
276
213
                                "$@"
277
214
                        if [ "$name" = component-mismatches ]; then
278
215
                                process-component-mismatches-diff \
279
216
                                        "$OUT/$name.txt" "$OUT/$name.txt.new"
280
217
                        fi
281
218
                        mv "$OUT/$name.txt.new" "$OUT/$name.txt"
282
 
                        dot -Tsvg -Gdpi=55 -o "$OUT/$name.svg" "$OUT/$name.dot"
 
219
                        dot -Tsvg -o "$OUT/$name.svg" "$OUT/$name.dot"
283
220
                }
284
221
                background pids component_mismatches component-mismatches
285
222
                background pids component_mismatches component-mismatches-proposed \
295
232
                        --csv-file "$OUT/architecture-mismatches.csv"
296
233
 
297
234
                background_wait pids
298
 
        else
299
 
                # Artifically bump $GERMINATE/germinate.output into the past
300
 
                touch -d @$(($NEW_GERMINATE_MTIME - 1)) $GERMINATE/germinate.output
301
235
        fi
302
236
fi