1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
#!/bin/bash
# vi: ts=4 noexpandtab
SAFE_DAYS=3
SAFE_COUNT=3
KEEP_BAD_BUILDS=2
SAFE_FILE=.save
BASES=( /srv/ec2-images/server )
RELS="precise trusty vivid wily xenial"
# keep ${SAVE_COUNT} builds or builds to ${SAVE_DAYS} days ago, whichever is greater.
export LANG=C
TEMP_D=""
error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
cleanup() {
[ -z "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] [ basedir [ basedir [ ... ] ] ]
clean up builds under basedir
if no basedir given, use:
${BASES[*]}
options:
-h | --help show usage
-n | --no-update-published do not update published info
--dry-run only report what would be done
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
short_opts="hn"
long_opts="help,dry-run,no-update-published"
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
bad_Usage
skip_modify=0
debug=""
update_published=1
info_dir=""
pt=( )
oargs=( "${0##*/}" "${@}" )
while [ $# -ne 0 ]; do
cur=${1}; next=${2};
case "$cur" in
--dry-run) debug=echo; pt=( "--dry-run" );;
-h|--help) Usage; exit 0;;
-n|--no-update-published) update_published=0;;
--) shift; break;;
esac
shift;
done
if [ $# -ne 0 ]; then
BASES=( "$@" )
fi
[ -n "${PUBLISH_BASE}" ] && info_dir="${PUBLISH_BASE}/query"
reldirs=( ); # like /srv/ec2-images/hardy
for base in "${BASES[@]}"; do
for x in ${RELS}; do
x=${base}/${x}
[ -d "${x}" ] || continue
reldirs[${#reldirs[@]}]=${x}
done
done
datedir="20[01][0-9][01][0-9][0123][0-9]"
safe_date=$(date --date "${SAFE_DAYS} days ago" +"%Y%m%d") ||
{ echo "failed to get date"; exit 1; }
topurge=( )
badbuilds=( )
for reldir in "${reldirs[@]}"; do
count=0;
saved=0;
bad=0
# loop backwards through the subdirs
rlist=$(for x in "${reldir}/"*; do echo "${x}"; done | LANG=C sort -r)
for dailyd in ${rlist}; do
bdir=${dailyd##*/}
# must be a dir
[ -d "${dailyd}" ] || continue
# must match datedir or datedir.[0-9]
case "${bdir}" in
${datedir}|${datedir}.[0-9]) :;;
*) continue;;
esac
ndate=${bdir%.*} # remove .1
# if a SAFE_FILE exists, skip the dir
[ ! -e "${dailyd}/${SAFE_FILE}" ] || { continue; }
# if it is a failed build, it should have 'log.stdout.stderr'
# if it does and it is older than safe_days, then remove it
# but dont count it (we only want to count good builds)
if [ -f "${dailyd}/log.stdout.stderr" ] &&
[ ! -f "${dailyd}/published-ec2-daily.txt" ]; then
bad=$((bad+1))
# if 'count' is non-zero, it means we've already
# seen a good build, and since we're counting backwards
# that one is newer than this one. So, with a good build
# around, we care less about keeping bad ones
if [ ${count} -ne 0 ]; then
badbuilds[${#badbuilds[@]}]=${dailyd};
else
# keep some bad builds around
if [ ${bad} -gt ${KEEP_BAD_BUILDS} ]; then
badbuilds[${#badbuilds[@]}]=${dailyd};
fi
fi
continue;
fi
# start counting so we know if we've hit SAFE_COUNT
count=$((${count}+1))
# if the directory is less than SAFE_DAYS old, leave it
[ "${ndate}" -lt "${safe_date}" ] || continue
# if we're over SAFE_COUNT, then add to purge
if [ "${count}" -gt "${SAFE_COUNT}" ]; then
if [ "${dailyd}" -ef "${reldir}/current" ]; then
error "ERROR: ${dailyd} == ${reldir}/current"
fail "exiting instead of removing current build"
fi
topurge[${#topurge[@]}]=${dailyd};
fi
done
done
if [ ${#topurge[@]} -ne 0 ]; then
error "removing daily builds: ${topurge[@]}"
${debug} rm -Rf "${topurge[@]}"
else
error "no daily builds to clean"
fi
if [ ${#badbuilds[@]} -ne 0 ]; then
error "removing bad builds: ${badbuilds[@]}"
${debug} rm -Rf "${badbuilds[@]}"
else
error "no bad builds to clean"
fi
# publicize-clean is in ec2-publishing-scripts and cleans up
# query data. It makes most sense to run it directly after cleaning
publicize-clean "${pt[@]}" || fail "failed to publicize-clean"
[ -z "${debug}" ] || exit 0
# trigger-sync is in ec2-publishing-scripts, and syncs
# the modified data from the build system to http hosting system
${debug} trigger-sync
exit 0
|