1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
|
#!/bin/bash
# Copyright (C) 2008-2015 Canonical, Ltd.
# Authors: Kees Cook <kees@ubuntu.com>
# Jamie Strandboge <jamie@ubuntu.com>
# License: GPLv3
#
# This script pulls down all the supported architectures (those that would
# appear in a sis-generate-usn run) Releases and Packages files.
#
# TODO: perform GPG tests (we're only using this info for component
# matching)
set -e
renice 10 -p $$ >/dev/null
ionice -c 2 -n 7 -p $$
help() {
cat <<EOM
Usage: packages_mirror [OPTIONS]
-t use timestamps (ie, don't update files if they have been
updated within the last day)
-f when using timestamps, force updating the files
-v verbose output
-V even more verbose output
-r RELEASE only update Ubuntu release
-A pull only release pocket for the devel release
-p pull only partner
EOM
}
find_devel_release() {
cvelib="`dirname $0`/cve_lib.py"
grep '^devel_release =' "$cvelib" | cut -d "'" -f 2 || true
}
use_timestamp="no"
force_timestamp="no"
verbosity_args="-q"
only_release=
devel_quick_pockets=
very_verbose=""
only_partner="no"
while getopts "AhftvVpr:" opt
do
case "$opt" in
f) force_timestamp="yes";;
t) use_timestamp="yes";;
v) verbosity_args="";;
V) verbosity_args=""
very_verbose="yes";;
A) devel_quick_pockets="yes";;
r) only_release="$OPTARG";;
p) only_partner="yes";;
h) help ; exit 0;;
?) help;;
esac
done
shift $(($OPTIND - 1))
#server=se.archive.ubuntu.com
#server=us.archive.ubuntu.com
server=archive.ubuntu.com
ports=ports.ubuntu.com
outPath=$(python -c 'import configobj, os.path; print configobj.ConfigObj(os.path.expanduser("~/.ubuntu-cve-tracker.conf"))["packages_mirror"]')
if [ -z "$outPath" ]; then
echo "'packages_mirror' not defined in ~/.ubuntu-cve-tracker.conf" >&2
exit 1
fi
mkdir -p "$outPath"
debianPath=$(python -c 'import configobj, os.path; print configobj.ConfigObj(os.path.expanduser("~/.ubuntu-cve-tracker.conf"))["debian_mirror"]')
if [ -z "$debianPath" ]; then
echo "'debian_mirror' not defined in ~/.ubuntu-cve-tracker.conf" >&2
exit 1
fi
mkdir -p "$debianPath"
partnerPath=$(python -c 'import configobj, os.path; print configobj.ConfigObj(os.path.expanduser("~/.ubuntu-cve-tracker.conf"))["partner_mirror"]')
if [ -z "$partnerPath" ]; then
echo "'partner_mirror' not defined in ~/.ubuntu-cve-tracker.conf" >&2
exit 1
fi
mkdir -p "$partnerPath"
partner_dir_excludes=$(PYTHONPATH="${PYTHONPATH:+$PYTHONPATH:}$(dirname $0)" python -c '#
import cve_lib, sys
s = ""
for r in cve_lib.eol_releases + ["breezy", "warty", "hoary"]:
if r in ["dapper", "breezy", "warty", "hoary"]: # http://archive.canonical.com/dists/ does not have these any more
continue
s += "dists/%s*/," % r
sys.stdout.write(s.rstrip(","))
')
function compressed_ext()
{
rel="$1"
# Use .gz in newer releases for now because Xenial and Yakkety only have
# .xz enabled for some pockets and the list of pockets seems to be changing
ext="gz"
# Prior releases without any .xz Packages files
bz2releases='^(precise|trusty|vivid)$'
if echo "$rel" | cut -d- -f1 | egrep -q "$bz2releases" ; then
ext="bz2"
fi
echo "$ext"
}
function spew_bin_lines()
{
rel="$1"
repo="$2"
arch="$3"
ext=$(compressed_ext "$rel")
echo "dists/$rel/$repo/binary-$arch/Release"
echo "dists/$rel/$repo/binary-$arch/Packages.$ext"
echo "dists/$rel/$repo/debian-installer/binary-$arch/Packages.$ext"
echo "dists/$rel/$repo/binary-$arch/Packages.gz"
echo "dists/$rel/$repo/debian-installer/binary-$arch/Packages.gz"
}
function spew_src_lines()
{
rel="$1"
repo="$2"
ext=$(compressed_ext "$rel")
echo "dists/$rel/$repo/binary-$arch/Release"
echo "dists/$rel/$repo/source/Release"
echo "dists/$rel/$repo/source/Sources.$ext"
echo "dists/$rel/$repo/source/Sources.gz"
}
function gen_packages()
{
release="$1"
arches="$2"
if [ -n "$only_release" ] && [ "$release" != "$only_release" ]; then
echo "Skipping '$release' (specified '-r $only_release')" >&2
return
fi
release_list="$release $release-updates $release-security $release-proposed $release-backports"
devel_release=`find_devel_release`
if [ -n "$devel_release" ] && [ "$devel_quick_pockets" = "yes" ] && [ "$devel_release" = "$release" ]; then
release_list="$release"
fi
for rel in $release_list
do
for repo in main restricted universe multiverse
do
for arch in $arches
do
spew_bin_lines $rel $repo $arch
done
spew_src_lines $rel $repo
done
done
}
function pull_packages()
{
filelist="$1"
url="$2"
if [ "$verbosity_args" != "-q" ]; then
echo "Fetching:"
cat "$filelist"
fi
count=1
num_tries=3
while [ "$count" -le "$num_tries" ]; do
# FIXME: how do I get both the return code and stdout into a variable?
log=$(mktemp -t rsync-XXXXXX)
set +e
if [ "$verbosity_args" != "-q" ]; then
rsync -rlptv --progress --files-from="$filelist" $url $outPath/ 2>&1 | tee "$log"
else
rsync -rlptq --files-from="$filelist" $url $outPath/ >"$log" 2>&1
fi
rc=$?
# keep trying if not successful (23 is missing files)
if [ $rc -eq 0 ] || [ $rc -eq 23 ]; then
break
fi
count=$(( $count + 1 ))
if [ "$verbosity_args" != "-q" ]; then
echo "Try: $count" >&2
fi
done
set -e
OUT=$(cat "$log" | egrep -v '(debian-installer|^rsync.*code 23)' || true)
rm -f "$log" || true
# Report errors, if any
if [ -n "$OUT" ]; then
echo "$OUT"
fi
# Some unknown error -- die
if [ $rc -ne 0 ] && [ $rc -ne 23 ]; then
return 1
fi
# Missing files (rc 23), die only if it's not a debian-installer path
if [ $rc -eq 23 ] && [ -n "$OUT" ]; then
return 1
fi
# Create uncompressed versions too
while read filename
do
if ! echo "$filename" | egrep -q '\.(bz2|xz)$' ; then
continue
fi
ext=$(echo "$filename" | awk -F. '{print $NF}')
src="$outPath/$filename"
dest_dir=$(dirname "$src")
dest_file=$(basename "$src" ."$ext")
dest="$dest_dir/$dest_file"
# Skip missing files (debian-installer)
if [ ! -r "$src" ]; then
continue
fi
if [ ! -f "$dest" ] || [ "$src" -nt "$dest" ]; then
case "$ext" in
bz2) bzcat "$src" > "$dest";;
xz) xzcat "$src" > "$dest";;
esac
fi
done < <(cat "$filelist")
}
# do_use_timestamp() returns '0' if timestamp doesn't exist or has been modified
# yesterday or more.
function do_use_timestamp() {
mtime=0
if [ -n "$2" ]; then
mtime="$2"
fi
if [ "$use_timestamp" = "no" ] || [ "$force_timestamp" = "yes" ]; then
return 0
elif [ ! -e "$1" ]; then
return 0
else
tmp=`find "$1" -mtime +$mtime`
if [ -n "$tmp" ]; then
return 0
fi
fi
local days=$((mtime + 1))
echo "'$1' exists and was modified within the last $days day(s). Skipping." >&2
return 1
}
if [ "$only_partner" = "no" ]; then
timestamp="${outPath}.timestamp"
if [ ! -e "${outPath}/dists" ] || do_use_timestamp "$timestamp" ; then
pull=$(mktemp -t packages-XXXXXX)
trap "rm -f $pull" EXIT HUP INT QUIT TERM
# Sync Packages for non-ports supported arches
if [ "$verbosity_args" != "-q" ]; then
echo "Generating arch lists"
fi
gen_packages precise "amd64 i386" >> "$pull"
gen_packages trusty "amd64 i386" >> "$pull"
gen_packages xenial "amd64 i386" >> "$pull"
gen_packages zesty "amd64 i386" >> "$pull"
gen_packages artful "amd64 i386" >> "$pull"
gen_packages bionic "amd64 i386" >> "$pull"
pull_packages "$pull" rsync://$server/ubuntu || echo "FAIL: supported architectures" >&2
cat /dev/null > "$pull"
# Sync Packages for ports arches
if [ "$verbosity_args" != "-q" ]; then
echo "Generating ports arch lists"
fi
gen_packages precise "powerpc armel armhf" >> "$pull"
gen_packages trusty "powerpc ppc64el armhf arm64" >> "$pull"
gen_packages xenial "powerpc ppc64el armhf arm64 s390x" >> "$pull"
gen_packages zesty "ppc64el armhf arm64 s390x" >> "$pull"
gen_packages artful "ppc64el armhf arm64 s390x" >> "$pull"
gen_packages bionic "ppc64el armhf arm64 s390x" >> "$pull"
pull_packages "$pull" rsync://$ports/ubuntu-ports || echo "FAIL: ports architectures" >&2
cat /dev/null > "$pull"
if [ "$use_timestamp" = "yes" ]; then
touch "$timestamp"
else
rm -f "$timestamp"
fi
fi
fi
#
# The wget trees
#
if [ -z "$very_verbose" ]; then
verbosity_args="-q"
fi
# TODO: add Debian support for -r
if [ -z "$only_release" ] && [ "$only_partner" = "no" ]; then
# Sync Sources from Debian testing
timestamp="${debianPath}.timestamp"
if [ ! -e "${debianPath}/dists" ] || do_use_timestamp "$timestamp" 6 ; then
cd "$debianPath"
for i in main contrib non-free
do
wget $verbosity_args -N -R '*=*' -R 'Contents*' -X '/debian/dists/testing/*/source/Sources.diff' -np -r ftp://ftp.debian.org/debian/dists/testing/$i/source/
done
ln -sf ftp.debian.org/debian/dists dists
# remove dangling symlink
rm -f ftp.debian.org/debian/dists/dists || true
if [ "$use_timestamp" = "yes" ]; then
touch "$timestamp"
else
rm -f "$timestamp"
fi
fi
fi
if [ -z "$only_release" ] || [ "$only_partner" = "yes" ]; then
# Sync Canonical Partner Repo
timestamp="${partnerPath}.timestamp"
if [ ! -e "${partnerPath}/dists" ] || do_use_timestamp "$timestamp" 6 ; then
cd "$partnerPath"
misc_partner_excludes="robots.txt,misc/,project/,icons/,ls-lR.gz"
wget $verbosity_args -N -R '*=*' -R 'Contents*' -X "${misc_partner_excludes},${partner_dir_excludes}" -np -r http://archive.canonical.com/dists/
ln -sf archive.canonical.com/dists dists
# remove dangling symlink
rm -f archive.canonical.com/dists/dists || true
# remove dists.[0-9] files
rm -f archive.canonical.com/dists.[0-9]*
if [ "$use_timestamp" = "yes" ]; then
touch "$timestamp"
else
rm -f "$timestamp"
fi
has_sources=`find archive.canonical.com/dists -name Sources`
if [ ! -z "$has_sources" ]; then
echo ""
echo "WARNING: $partnerPath has Sources files! These should be removed now that -partner uses Sources.gz"
fi
fi
fi
|