1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
|
#!/bin/bash
# vi: ts=4 noexpandtab
TMPD=""
VERBOSITY=0
YYYYMMDD="'s,.*\(20[0-9][0-9]\)-\{0\,1\}\([01][0-9]\)-\{0\,1\}\([0-3][0-9]\).*,\1-\2-\3,p'"
DATA_BUCKET="ubuntu-data/pubinfo"
UNPUB_PATH_FMT="${DATA_BUCKET}/%r.%i.unpublished"
error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
debugp() {
[ "${1}" -gt "${VERBOSITY}" ] && return
shift;
errorp "${@}"
}
Usage() {
# TODO:
# for the sandbox builds, simply privatize things that appear older
# than a certain age
# -p | --privatize-age privatize builds older than D
# [names must have YYYYMMDD format]
cat <<EOF
Usage: ${0##*/} [ options ] region regexp
-n | --dry-run only report what would be done
-k | --keep-public N keep at most N builds matching
-r | --remove-unpub-age D remove builds that were unpublished D days ago
Example:
${0##*/} --dry-run --keep-public 5 --remove-unpub-age 30 \\
us-east-1 "^ubuntu-images-testing-us/ubuntu-karmic-daily-amd64"
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
cleanup() {
[ -n "${TMPD}" -a -d "${TMPD}" ] && rm -Rf "${TMPD}"
return 0
}
cache_make_private() {
local file=${1} id="" tab=" "
local tok="[^${tab}]*${tab}" local m_begin="" t=""
shift
for id in "${@}"; do
m_begin="IMAGE${tab}${id}${tab}${tok}${tok}${tok}"
sed -i "s/\(${m_begin}\)${tok}\(.*\)$/\1private${tab}\2/" "${file}" &&
t=$(awk "-F\t" '$2 == id { print $6 }' "id=$id" "${file}") &&
[ "${t}" = "private" ] ||
return 1
done
}
short_opts="hk:nr:v"
long_opts="dry-run,help,keep-public:,remove-unpub-age:,verbose"
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
bad_Usage
dry_run=0;
remove_unpub=0;
keep_public=0
VERBOSITY=0
im_data=""
while [ $# -ne 0 ]; do
cur=${1}; next=${2};
case "$cur" in
--) shift; break;;
-h|--help) Usage; exit 0;;
-n|--dry-run) dry_run=1; VERBOSITY=$(($VERBOSITY+1));;
-k|--keep-public) keep_public=${next}; shift;;
-r|--remove-unpub-age) remove_unpub=${next}; shift;;
-v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
esac
shift;
done
region=${1}
regex=${2}
[ $# -lt 2 ] && bad_Usage "must provide region, regex"
[ $# -gt 2 ] && bad_Usage "too many args: $*"
for x in ${remove_unpub} ${keep_public}; do
case "$x" in
*[^0-9]*) fail "keep or remove must be numeric";;
esac
done
[ "${remove_unpub}" != "0" -o "${keep_public}" != "0" ] ||
fail "nothing to do. specify remove-privatized or keep-public"
TMPD=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX) ||
fail "failed to make tempdir"
trap cleanup EXIT
export XC2_XIMAGES_CACHE_D=${XC2_XIMAGES_CACHE_D:-"$TMPD/ximgcache"}
# get a list of all images, and then sort it by basename
im_data="${TMPD}/describe-images.${region}"
xc2 ximages describe-images --all --region "${region}" > "${im_data}" ||
fail "failed to get list of images in ${region}"
im_sort="${TMPD}/describe-images.${region}.sort"
awk '-F\t' 'BEGIN { OFS=FS; } $1 == "IMAGE" {
bname=$3; sub(/.*[\/]/,"",bname); print bname, $0; }' "${im_data}" |
LANG=C sort -k1 -r | sed 's,[^\t]*\t,,' > "${im_sort}"
if [ ${keep_public} -gt 0 ]; then
matching_pub="${TMPD}/matching.pub"
awk '-F\t' '$6 == "public" && $3 ~ regex { print $2,$13,$3 }' \
"regex=${regex}" "${im_sort}" > "${matching_pub}" ||
fail "failed to get matching info"
n=0
while read id img_type name; do
n=$((n+1))
if [ "${n}" -gt "${keep_public}" ]; then
debugp 1 "%s\n" "privatize ${region} ${id} ${name}"
[ "${dry_run}" -eq 0 ] || continue
privatize-image --follow --skip-referenced \
--unpub-path-fmt="${UNPUB_PATH_FMT}" \
"${region}" "${id}" ||
fail "failed to privatize ${id} ${name}"
cache_make_private "${im_data}" "${id}" ||
fail "failed to make ${id} private in cache"
else
debugp 1 "%s\n" "keep ${region} ${id} ${name}"
[ "${dry_run}" -eq 0 ] || continue
fi
done < "${matching_pub}"
fi
if [ "${remove_unpub}" -gt 0 ]; then
matching_pri="${TMPD}/matching.pri"
# this list uses the original 'im_sort' created from above
# as such, it will not include images that were made private above
# there is no need for it to, as those are certainly not older than
# remove_unpub
awk '-F\t' '$6 == "private" && $3 ~ regex { print $2,$13,$3 }' \
"regex=${regex}" "${im_sort}" > "${matching_pri}" ||
fail "failed to get matching private info"
today=$(date +%s --date "00:00 UTC") &&
purge_unix=$((${today}-(${remove_unpub}*24*60*60))) ||
fail "failed to get date for ${remove_unpub} days ago"
uptmp="${TMPD}/unpublished"
while read id img_type name; do
# TODO: This is really broken in the case where
# you have an instance-store manifest registered more than once
# none of the Ubuntu tools do that, but really, this ".unpublished".
# file should represent that somehow.
upname=${UNPUB_PATH_FMT//%r/${region}}
upname=${upname//%i/${id}}
if ! xc2 s3cmd get "s3://${upname}" - > "${uptmp}" 2>/dev/null; then
debugp 1 "#Warning: %s was private, but had no unpub file in %s. privatizing\n" \
"${upname}" "${id}"
debugp 1 "%s\n" "privatize ${region} ${id} ${name}"
privatize-image --follow --skip-referenced \
--unpub-path-fmt="${UNPUB_PATH_FMT}" \
"${region}" "${id}" ||
fail "failed to privatize ${id} ${name}"
# these were already private, so this should do nothing
# but just do this to be consistent
cache_make_private "${im_data}" "${id}" ||
fail "failed to make ${id} private in ${cache}"
continue;
fi
read unpub_d < "${uptmp}" &&
unpub_unix=$(date +%s --date "${unpub_d}") ||
fail "failed to read date from ${name}.unpublished"
if [ "${unpub_unix}" -gt "${purge_unix}" ]; then
debugp 1 "%s\n" "keep_pri ${region} ${id} ${name}"
continue
else
debugp 1 "%s\n" "remove ${region} ${id} ${name}"
[ "${dry_run}" -eq 0 ] || continue;
remove-image --unpub-path-fmt="${UNPUB_PATH_FMT}" \
"${region}" "${id}" ||
error "failed to remove ${region} ${id} ${name}"
fi
done < "${matching_pri}"
fi
|