« back to all changes in this revision
Viewing changes to build-lb-ec2-image
- browse files at revision 644
- compare with another revision
- download diff
- download tarball
- view history from revision 644
- Committer: Ben Howard
- Date: 2014-08-22 17:11:27 UTC
- Revision ID: ben.howard@ubuntu.com-20140822171127-r8b67lxfwq9xe0ke
Whitespace changes. Added the ability to build images from -proposed.
- files modified:
- build-ec2-image
added
removed
build-lb-ec2-image
17
17
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
18
18
debug() { error "$(date -R):" "$@"; }
19
19
cleanup() {
20
if [ -n "${TEMP_D}" -a -d "${TEMP_D}" ]; then
21
umount_r "${TEMP_D}" && rm -rf "${TEMP_D}" || return 1
22
fi
23
if [ -n "${FINAL_DIR}" -a -d "${FINAL_DIR}" -a -n "${CHOWN_OPTS}" ]; then
24
chown -R "${CHOWN_OPTS}" "${FINAL_DIR}"
25
fi
20
if [ -n "${TEMP_D}" -a -d "${TEMP_D}" ]; then
21
umount_r "${TEMP_D}" && rm -rf "${TEMP_D}" || return 1
22
fi
23
if [ -n "${FINAL_DIR}" -a -d "${FINAL_DIR}" -a -n "${CHOWN_OPTS}" ]; then
24
chown -R "${CHOWN_OPTS}" "${FINAL_DIR}"
25
fi
26
26
}
27
27
28
28
usage() {
31
31
Uses customized live build to produce Cloud Images
32
32
33
33
Required Flags:
34
--config Config
34
--config Config
35
35
36
36
Optional
37
37
--arch Arch type, i.e. amd64 or i386, default all
42
42
--owner Set the owner of the resulting files
43
43
--copy Copy the final image for debugig to location
44
44
--manifest Where to drop the manfiest file
45
--livepath Location of live-build scripts
46
--serial image serial number, defaults to YYYYMMDD
47
--hookscript Add a script to hooks script section
48
--ec2-version Destription of EC2 image
45
--livepath Location of live-build scripts
46
--serial image serial number, defaults to YYYYMMDD
47
--hookscript Add a script to hooks script section
48
--ec2-version Destription of EC2 image
49
--propsoed Build from propoposed
49
50
EOF
50
51
}
51
52
52
53
# Parse options
53
54
54
55
short_opts="d,l,h"
55
long_opts="arch:,dest:,part:,debug,owner:,copy:,manifest:,name:,suite:,version:,type:,rootlabel:,moniker:,image:,rootsize:,livepath:,serial:,hookscript:,ec2-version:,config:"
56
long_opts="arch:,dest:,part:,debug,owner:,copy:,manifest:,name:,suite:,version:,type:,rootlabel:,moniker:,image:,rootsize:,livepath:,serial:,hookscript:,ec2-version:,config:,proposed"
56
57
getopt_out=$(getopt --name "${0##*/}" \
57
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
58
eval set -- "${getopt_out}" ||
59
bad_Usage
58
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
59
eval set -- "${getopt_out}" ||
60
bad_Usage
60
61
61
62
arch_types="amd64 i386"
62
63
finaldir="/tmp/live-build-$(date +%Y%m%d)_$(uuidgen)" # This is usually overriden at runtime
77
78
root_size=""
78
79
hookscript=""
79
80
config=""
81
proposed=0
80
82
serial=$(date +%Y%m%d)
81
83
live_path="${LIVE_BUILD_PATH}"
82
84
83
85
while [ $# -ne 0 ]; do
84
86
case $1 in
85
--config) config="$2"; shift;;
86
--arch) arch_types="$2"; shift;;
87
--dest) finaldir="$2"; shift;;
88
--part) partfile="$2"; shift;;
89
-d|--debug) debug=1; shift;;
90
--owner) owner="$2"; shift;;
91
--manifest) manifest="$2"; shift;;
92
--copy) copy_final="$2"; shift;;
93
--name|--suite) suite="$2"; shift;;
94
--version) version="$2"; shift;;
95
--type) build_type="$2"; shift;;
96
--rootlabel) root_label="$2"; shift;;
97
--moniker) moniker="$2"; shift;;
98
--image) imagename="$2"; shift;;
99
--rootsize) root_size="$2"; shift;;
100
--livepath) live_path="$2"; shift;;
101
--serial) serial="$2"; shift;;
102
--hookscript) hookscript="$2"; shift;;
103
--ec2-version) description="$2"; shift;;
87
--config) config="$2"; shift;;
88
--arch) arch_types="$2"; shift;;
89
--dest) finaldir="$2"; shift;;
90
--part) partfile="$2"; shift;;
91
-d|--debug) debug=1; shift;;
92
--owner) owner="$2"; shift;;
93
--manifest) manifest="$2"; shift;;
94
--copy) copy_final="$2"; shift;;
95
--name|--suite) suite="$2"; shift;;
96
--version) version="$2"; shift;;
97
--type) build_type="$2"; shift;;
98
--rootlabel) root_label="$2"; shift;;
99
--moniker) moniker="$2"; shift;;
100
--image) imagename="$2"; shift;;
101
--rootsize) root_size="$2"; shift;;
102
--livepath) live_path="$2"; shift;;
103
--serial) serial="$2"; shift;;
104
--hookscript) hookscript="$2"; shift;;
105
--ec2-version) description="$2"; shift;;
106
--proposed) proposed=1; shift;;
104
107
105
-h|--help) Usage; exit 0;;
106
--) shift; break;;
107
esac
108
shift;
108
-h|--help) Usage; exit 0;;
109
--) shift; break;;
110
esac
111
shift;
109
112
done
110
113
111
114
#Done setting options
129
132
# umount_r(mp) : unmount any filesystems under r
130
133
# this is useful to unmount a chroot that had sys, proc ... mounted
131
134
umount_r() {
132
local p
133
for p in "$@"; do
134
[ -n "$p" ] || continue
135
tac /proc/mounts | sudo sh -c '
136
p=$1
137
while read s mp t opt a b ; do
138
[ "${mp}" = "${p}" -o "${mp#${p}/}" != "${mp}" ] ||
139
continue
140
umount "$mp" || exit 1
141
done
142
exit 0' umount_r "${p%/}"
143
[ $? -eq 0 ] || return
144
done
135
local p
136
for p in "$@"; do
137
[ -n "$p" ] || continue
138
tac /proc/mounts | sudo sh -c '
139
p=$1
140
while read s mp t opt a b ; do
141
[ "${mp}" = "${p}" -o "${mp#${p}/}" != "${mp}" ] ||
142
continue
143
umount "$mp" || exit 1
144
done
145
exit 0' umount_r "${p%/}"
146
[ $? -eq 0 ] || return
147
done
145
148
}
146
149
147
150
pids_using() {
159
162
}
160
163
161
164
force_unuse_d() {
162
local d="$1" i=0 tries=10 pids=""
163
while pids_using "$d" && pids=${_RET} && [ -n "${pids}" ] &&
164
[ $i -lt $tries ] && i=$(($i+1)); do
165
error "killing pids occupying ${d}:"
166
sudo ps -p ${pids} 1>&2
167
sudo kill -9 ${pids}
168
sleep 2
169
pids=""
170
done
171
# if there are pids around, then we failed
172
[ -z "${pids}" ]
165
local d="$1" i=0 tries=10 pids=""
166
while pids_using "$d" && pids=${_RET} && [ -n "${pids}" ] &&
167
[ $i -lt $tries ] && i=$(($i+1)); do
168
error "killing pids occupying ${d}:"
169
sudo ps -p ${pids} 1>&2
170
sudo kill -9 ${pids}
171
sleep 2
172
pids=""
173
done
174
# if there are pids around, then we failed
175
[ -z "${pids}" ]
173
176
}
174
177
175
178
get_owner_chown_opts() {
176
local owner="$1" opts="${owner}:"
177
shift;
178
if [ -z "${owner}" -a -n "${SUDO_UID}" ]; then
179
opts="${SUDO_UID}:${SUDO_GID}"
180
elif [ -z "${owner}" -a -n "${SUDO_USER}" ]; then
181
opts="${SUDO_USER}:$(id ${SUDO_USER} -g)"
182
elif [ -n "${owner}" ]; then
183
opts="$(id -u ${owner}):$(id -g ${owner})"
184
else
185
opts="$(id -u):$(id -g)"
186
fi
187
_RET=${opts}
188
return
179
local owner="$1" opts="${owner}:"
180
shift;
181
if [ -z "${owner}" -a -n "${SUDO_UID}" ]; then
182
opts="${SUDO_UID}:${SUDO_GID}"
183
elif [ -z "${owner}" -a -n "${SUDO_USER}" ]; then
184
opts="${SUDO_USER}:$(id ${SUDO_USER} -g)"
185
elif [ -n "${owner}" ]; then
186
opts="$(id -u ${owner}):$(id -g ${owner})"
187
else
188
opts="$(id -u):$(id -g)"
189
fi
190
_RET=${opts}
191
return
189
192
}
190
193
191
194
builder() {
192
#Run live-build. Hopefully this works
193
local base_d=${1} arch="${2}" imagename="${3}" out_d="${4}"
194
local chown_opts=${5} copy_final=${6}
195
local ret=0 start_d="$PWD"
196
197
debug "Running builder"
198
debug " builder1) Image will be named ${imagename}"
199
debug " builder1) Building for ${arch}"
200
debug " builder1) build location is "${base_d}""
201
202
local lbb="${base_d}/live-build" lb_env="" lbcmd=""
203
lb_env=( "LB_BASE=${lbb}" "PATH=${lbb}/scripts/build:${PATH}"
204
"${lbb}/scripts/build/lb" build )
205
lb_cmd=( sudo "${lb_env[@]}" "${lbb}/scripts/build/lb" build )
206
207
debug " builder2) cmd: " "${lb_cmd[@]}"
208
debug " builder3) Starting build prcoess...this might take a while"
209
210
debug " _______ LIVE-BUILD OUTPUT _______"
211
cd "${base_d}"
212
"${lb_cmd[@]}"; ret=$?
213
cd "${start_d}"
214
debug " _______ END LIVE-BUILD OUTPUT _____"
215
216
if ! umount_r "${base_d}/chroot" "${base_d}/cloud"; then
217
error "failed to unmount directories under ${base_d}";
218
force_unuse_d "${base_d}/chroot" "${base_d}/cloud" &&
219
umount_r "${base_d}/chroot" "${base_d}/cloud" &&
220
error " recovered via force_unuse_d"
221
return 1;
222
fi
223
224
# if live-build failed, return failure
225
[ ${ret} -eq 0 ] || { error "live-build failed"; return $ret; }
226
227
# Move the image into place
228
[ -e "${base_d}/binary-raw.img" ] ||
229
{ error "binary-raw.img did not exist in output directory"; return 1; }
230
mv "${base_d}/binary-raw.img" "${out_d}/${imagename}"
231
232
if [ ! -z "${copy_final}" ]; then
233
cp "${base_d}/binary-raw.img" "${copy}"
234
debug " builder3) Copied final image to ${copy_final}"
235
fi
236
237
debug " builder3) Binary cloud image can be found at ${out_d}/${imagename}"
238
239
# Pull out the binary.packages
240
if [ -e "${base_d}/binary.packages" ]; then
241
cp "${base_d}/binary.packages" "${manifest}" || fail "Unable to copy ${base_d}/binary.packages to ${manifest}"
242
sed -i "s|\t| |g" "${manifest}" || fail "Unable to normalize format of manifest file"
243
else
244
error " builder3) !! ERROR !! Unable to find manifest"
245
fi
246
247
# Clean up after ourselves
248
if [ -n "${chown_opts}" ]; then
249
sudo chown -R "${chown_opts}" "${base_d}" "${out_d}" ||
250
{ error "failed to change ownership to ${owner}" ; return 1; }
251
fi
195
#Run live-build. Hopefully this works
196
local base_d=${1} arch="${2}" imagename="${3}" out_d="${4}"
197
local chown_opts=${5} copy_final=${6}
198
local ret=0 start_d="$PWD"
199
200
debug "Running builder"
201
debug " builder1) Image will be named ${imagename}"
202
debug " builder1) Building for ${arch}"
203
debug " builder1) build location is "${base_d}""
204
205
local lbb="${base_d}/live-build" lb_env="" lbcmd=""
206
lb_env=( "LB_BASE=${lbb}" "PATH=${lbb}/scripts/build:${PATH}"
207
"${lbb}/scripts/build/lb" build )
208
lb_cmd=( sudo "${lb_env[@]}" "${lbb}/scripts/build/lb" build )
209
210
debug " builder2) cmd: " "${lb_cmd[@]}"
211
debug " builder3) Starting build prcoess...this might take a while"
212
213
debug " _______ LIVE-BUILD OUTPUT _______"
214
cd "${base_d}"
215
"${lb_cmd[@]}"; ret=$?
216
cd "${start_d}"
217
debug " _______ END LIVE-BUILD OUTPUT _____"
218
219
if ! umount_r "${base_d}/chroot" "${base_d}/cloud"; then
220
error "failed to unmount directories under ${base_d}";
221
force_unuse_d "${base_d}/chroot" "${base_d}/cloud" &&
222
umount_r "${base_d}/chroot" "${base_d}/cloud" &&
223
error " recovered via force_unuse_d"
224
return 1;
225
fi
226
227
# if live-build failed, return failure
228
[ ${ret} -eq 0 ] || { error "live-build failed"; return $ret; }
229
230
# Move the image into place
231
[ -e "${base_d}/binary-raw.img" ] ||
232
{ error "binary-raw.img did not exist in output directory"; return 1; }
233
mv "${base_d}/binary-raw.img" "${out_d}/${imagename}"
234
235
if [ ! -z "${copy_final}" ]; then
236
cp "${base_d}/binary-raw.img" "${copy}"
237
debug " builder3) Copied final image to ${copy_final}"
238
fi
239
240
debug " builder3) Binary cloud image can be found at ${out_d}/${imagename}"
241
242
# Pull out the binary.packages
243
if [ -e "${base_d}/binary.packages" ]; then
244
cp "${base_d}/binary.packages" "${manifest}" || fail "Unable to copy ${base_d}/binary.packages to ${manifest}"
245
sed -i "s|\t| |g" "${manifest}" || fail "Unable to normalize format of manifest file"
246
else
247
error " builder3) !! ERROR !! Unable to find manifest"
248
fi
249
250
# Clean up after ourselves
251
if [ -n "${chown_opts}" ]; then
252
sudo chown -R "${chown_opts}" "${base_d}" "${out_d}" ||
253
{ error "failed to change ownership to ${owner}" ; return 1; }
254
fi
252
255
253
256
}
254
257
267
270
268
271
for arch in ${arch_types}
269
272
do
270
work_d="${TEMP_D}/$arch"
271
conf_d="${work_d}/config"
272
mkdir -p "${work_d}"
273
274
275
# Dynamically configure LB
276
{
277
debug "Dynamically configuring live-build"
278
cd ${work_d}
279
280
lbb="${work_d}/live-build"
281
lbb_path="${lbb}/scripts/build"
282
mkdir -p "${lbb_path}"
283
284
rsync -a "${live_path}/" "${work_d}/live-build" ||
285
fail "failed to copy ${live_path}"
286
287
new_path="PATH=${live_path}/scripts/build:${PATH}"
288
289
# %d-> distro, %l-> hard disk label, %s-> size, %q-> qemu-static location
290
lbcmd=( "${lb_conf[@]}" )
291
lbcmd=( "${lbcmd[@]//%d/${suite}}")
292
lbcmd=( "${lbcmd[@]//%l/${root_fs_label}}")
293
lbcmd=( "${lbcmd[@]//%s/${root_fs_size}}")
294
lbcmd=( "${lbcmd[@]//%P/${lbb_path}}")
295
lbcmd=( "${lbcmd[@]//%L/${lbb}}")
296
lbcmd=( "${lbcmd[@]//%x/${new_path}}")
297
lbcmd=( "${lbcmd[@]//%A/${arch}}")
298
299
# Discover qemu location and set it for arm images
300
[[ "${arch}" =~ (arm|arm64) ]] && {
301
case ${arch} in
302
armel|armhf) q_arch="arm";;
303
arm64) q_arch="aarch64";;
304
*) q_arch="$arch";;
305
esac
306
debug "looking for qemu-${q_arch}-static"
307
308
[ -e "${HOME}/ec2-daily/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ] &&
309
qemu_exec="${HOME}/ec2-daily/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ||
310
[ -e "${lb_config_src_loc}/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ] &&
311
qemu_exec="${lb_config_src_loc}/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ||
312
qemu_exec="$(which qemu-${q_arch}-static)" ||
313
fail "Unable to locate qemu-${q_arch}-static"
314
315
lbcmd=( "${lbcmd[@]//%q/${qemu_exec}}" )
316
debug "found ${qemu_exec}"
317
debug "debootstrap will be foreign"
318
}
319
320
321
debug "lb_config commmand is: ${lbcmd[@]}"
322
sudo "${lbcmd[@]}" ||
323
fail "Failed to dynammicaly configure live-build"
324
325
}
326
327
# Update ubuntu-cloud definition for precise
328
[ "${suite}" \< "precise" ] || {
329
echo "update-notifier-common" >> "${work_d}/live-build/package-lists/ubuntu-cloud" ||
330
fail "Unable append update-notifier-common to package definition"
331
}
332
333
# make config tempdir
334
rsync -a "${lb_config_src_loc}/common/" "${conf_d}" ||
335
fail "failed to copy common hooks"
336
337
# seed directories
338
mkdir -p "${conf_d}/includes.chroot/etc/cloud"
339
340
# write build.info
341
echo "build_name: ${build_type}" >> "${conf_d}/includes.chroot/etc/cloud/build.info" &&
342
echo "serial: ${serial}" >> "${conf_d}/includes.chroot/etc/cloud/build.info" ||
343
fail "Unable to write etc/cloud/build.info"
344
345
# write ec2-version
346
echo "${description}" >> "${conf_d}/includes.chroot/etc/ec2_version" ||
273
work_d="${TEMP_D}/$arch"
274
conf_d="${work_d}/config"
275
mkdir -p "${work_d}"
276
277
278
# Dynamically configure LB
279
{
280
debug "Dynamically configuring live-build"
281
cd ${work_d}
282
283
lbb="${work_d}/live-build"
284
lbb_path="${lbb}/scripts/build"
285
mkdir -p "${lbb_path}"
286
287
rsync -a "${live_path}/" "${work_d}/live-build" ||
288
fail "failed to copy ${live_path}"
289
290
new_path="PATH=${live_path}/scripts/build:${PATH}"
291
292
# %d-> distro, %l-> hard disk label, %s-> size, %q-> qemu-static location
293
lbcmd=( "${lb_conf[@]}" )
294
lbcmd=( "${lbcmd[@]//%d/${suite}}")
295
lbcmd=( "${lbcmd[@]//%l/${root_fs_label}}")
296
lbcmd=( "${lbcmd[@]//%s/${root_fs_size}}")
297
lbcmd=( "${lbcmd[@]//%P/${lbb_path}}")
298
lbcmd=( "${lbcmd[@]//%L/${lbb}}")
299
lbcmd=( "${lbcmd[@]//%x/${new_path}}")
300
lbcmd=( "${lbcmd[@]//%A/${arch}}")
301
302
# Build the chroot from -proposed
303
[ "${proposed}" -eq 1 ] && lbcmd+=("--proposed=true")
304
305
# Discover qemu location and set it for arm images
306
[[ "${arch}" =~ (arm|arm64) ]] && {
307
case ${arch} in
308
armel|armhf) q_arch="arm";;
309
arm64) q_arch="aarch64";;
310
*) q_arch="$arch";;
311
esac
312
debug "looking for qemu-${q_arch}-static"
313
314
[ -e "${HOME}/ec2-daily/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ] &&
315
qemu_exec="${HOME}/ec2-daily/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ||
316
[ -e "${lb_config_src_loc}/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ] &&
317
qemu_exec="${lb_config_src_loc}/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ||
318
qemu_exec="$(which qemu-${q_arch}-static)" ||
319
fail "Unable to locate qemu-${q_arch}-static"
320
321
lbcmd=( "${lbcmd[@]//%q/${qemu_exec}}" )
322
debug "found ${qemu_exec}"
323
debug "debootstrap will be foreign"
324
}
325
326
327
debug "lb_config commmand is: ${lbcmd[@]}"
328
sudo "${lbcmd[@]}" ||
329
fail "Failed to dynammicaly configure live-build"
330
331
}
332
333
# Update ubuntu-cloud definition for precise
334
[ "${suite}" \< "precise" ] || {
335
echo "update-notifier-common" >> "${work_d}/live-build/package-lists/ubuntu-cloud" ||
336
fail "Unable append update-notifier-common to package definition"
337
}
338
339
# make config tempdir
340
rsync -a "${lb_config_src_loc}/common/" "${conf_d}" ||
341
fail "failed to copy common hooks"
342
343
# seed directories
344
mkdir -p "${conf_d}/includes.chroot/etc/cloud"
345
346
# write build.info
347
echo "build_name: ${build_type}" >> "${conf_d}/includes.chroot/etc/cloud/build.info" &&
348
echo "serial: ${serial}" >> "${conf_d}/includes.chroot/etc/cloud/build.info" ||
349
fail "Unable to write etc/cloud/build.info"
350
351
# write ec2-version
352
echo "${description}" >> "${conf_d}/includes.chroot/etc/ec2_version" ||
347
353
fail "Unable to write ${conf_d}/includes.chroot/etc/ec2_version"
348
354
349
[ -z "${hookscript}" ] || {
350
local_hook="${conf_d}/hooks/999hookscript.chroot"
351
352
cp -a "${hookscript}" "${local_hook}" ||
353
fail "Unable to copy ${hookscript} to ${local_hook}"
354
355
debug "Populated hookscript to: ${conf_d}/hooks"
356
}
357
358
builder "${work_d}" "${arch}" "${imagename}" "${finaldir}" \
359
"${chown_opts}" "${copy_final}" ||
360
fail "failed to build for ${suite}"
361
362
rm -Rf "${work_d}" || fail "Unable to clean ${temp_d}"
355
[ -z "${hookscript}" ] || {
356
local_hook="${conf_d}/hooks/999hookscript.chroot"
357
358
cp -a "${hookscript}" "${local_hook}" ||
359
fail "Unable to copy ${hookscript} to ${local_hook}"
360
361
debug "Populated hookscript to: ${conf_d}/hooks"
362
}
363
364
builder "${work_d}" "${arch}" "${imagename}" "${finaldir}" \
365
"${chown_opts}" "${copy_final}" ||
366
fail "failed to build for ${suite}"
367
368
rm -Rf "${work_d}" || fail "Unable to clean ${temp_d}"
363
369
done
364
370
365
371
# Cleanup
366
372
cleanup && TEMP_D="" && FINAL_DIR="" && CHOWN_OPTS="" ||
367
fail "failed to cleanup"
373
fail "failed to cleanup"
368
374
369
375
debug "Images and logs can be found at "${finaldir}""
Loggerhead is a web-based interface for Breezy
Version: 2.0.1