|
112
by Scott Moser
add non-working version of ec2-image2ebs |
1 |
#!/bin/bash
|
2 |
# vi: ts=4 noexpandtab
|
|
3 |
||
4 |
TEMP_D="" |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
5 |
VOLUME_ID="" |
6 |
VOLUME_ATTACHED=0 |
|
7 |
SNAPSHOT_ID="" |
|
|
120
by Scott Moser
get rid of the ssh config in ec2-image2ebs. |
8 |
SSH_USER=${SSH_USER:-ubuntu} |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
9 |
REGARGS=( ) |
|
370
by Scott Moser
ec2-image2ebs: retry ssh verify connection. more quiet apt-get. |
10 |
IID="" |
|
115
by Scott Moser
some '--region' argument fixes, use $SSH rather than ssh |
11 |
SSH=${SSH:-"ssh"} |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
12 |
|
|
369
by Scott Moser
ec2-image2ebs, run-instance-and-wait: add prefix, make more quiet |
13 |
error() { echo ${PREFIX:+"${PREFIX}"} "$@" 1>&2; } |
|
556.1.1
by Ben Howard
Inital work |
14 |
fail() { |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
15 |
isfalse ${SNAPSHOT_ID} || delete_snapshot "${SNAPSHOT_ID}" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
16 |
[ $# -eq 0 ] || error "$@"; |
|
370
by Scott Moser
ec2-image2ebs: retry ssh verify connection. more quiet apt-get. |
17 |
local out="" |
|
384
by Scott Moser
run-instance-and-wait ec2-image2ebs, use info-instances |
18 |
[ -z "$IID" ] || out=$(xc2 info-instances "${REGARGS[@]}" "${IID}") |
|
370
by Scott Moser
ec2-image2ebs: retry ssh verify connection. more quiet apt-get. |
19 |
[ -n "$out" ] || error "$out" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
20 |
exit 1;
|
21 |
}
|
|
22 |
debug() { |
|
23 |
local level=${1} |
|
24 |
shift;
|
|
25 |
[ "${level}" -gt "${VERBOSITY}" ] && return |
|
|
634
by Robert Jennings
Remove "error" from debug output |
26 |
echo "$(date): ${@}" 1>&2 |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
27 |
}
|
|
406.1.2
by Scott Moser
remove race condition in detach_vol |
28 |
get_volstate() { |
29 |
local vol="$1" tmp="" |
|
30 |
tmp=$(mktemp "$TEMP_D/describe_vol_out.$vol.XXXXXX") || |
|
31 |
return 1
|
|
|
604
by Ben Howard
Drop all use of EUCA tools due to signing issues |
32 |
if xc2 describe-volumes "${REGARGS[@]}" "$vol" > "$tmp"; then |
|
406.1.2
by Scott Moser
remove race condition in detach_vol |
33 |
_RET=$(awk '-F\t' '$1 == "VOLUME" { print $6 }' "$tmp") |
34 |
rm -f "$tmp"
|
|
35 |
return 0
|
|
36 |
fi
|
|
|
604
by Ben Howard
Drop all use of EUCA tools due to signing issues |
37 |
error "ec2-describe-volumes $vol failed" |
|
406.1.2
by Scott Moser
remove race condition in detach_vol |
38 |
rm -f "$tmp"
|
39 |
_RET="" |
|
40 |
}
|
|
41 |
||
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
42 |
detach_vol() { |
|
587
by Ben Howard
Be aggressive about calling detach on volumes |
43 |
local vol="${1}" log=${2:-/dev/null} max=${3:-20} state="" i=0 ret=1; |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
44 |
debug 2 "detaching ${vol}"
|
|
589
by Robert Jennings
Fix until; do; done syntax |
45 |
until [ "${i}" -gt "${max}" -o "$state" = "available" ]; do |
|
587
by Ben Howard
Be aggressive about calling detach on volumes |
46 |
i=$(($i + 1)) |
47 |
retry 3 30 get_volstate "$vol" && state="${_RET}" |
|
48 |
debug 2 "volume state after pass $i/$max, state=$state"
|
|
49 |
case "${state}" in |
|
|
590
by Robert Jennings
Missing ;; for case statement |
50 |
available) debug 2 "${vol} is available, returning 0"; ret=0;; |
|
587
by Ben Howard
Be aggressive about calling detach on volumes |
51 |
in-use) debug 2 "${vol} is attached calling for detach"; |
52 |
retry 3 30 xc2 detach-volume "${REGARGS[@]}" "${vol}" > ${log}; |
|
53 |
sleep 60;; |
|
54 |
esac
|
|
|
326
by Scott Moser
make detach_vol return false if it doesn't result in detached volume |
55 |
done
|
|
587
by Ben Howard
Be aggressive about calling detach on volumes |
56 |
error "after $i waits, $vol, state=$state" |
|
326
by Scott Moser
make detach_vol return false if it doesn't result in detached volume |
57 |
return $ret |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
58 |
}
|
59 |
||
60 |
delete_vol() { |
|
61 |
local vol=${1} |
|
62 |
debug 2 "deleting ${vol}"
|
|
|
327
by Ben Howard
Added support to define specific EC2 regions via EC2_ALL_REGIONS environmental variable |
63 |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
64 |
xc2 delete-volume "${REGARGS[@]}" "${vol}" > ${2:-/dev/null} || |
65 |
{ error "failed to delete ${vol}"; return 1; } |
|
66 |
}
|
|
67 |
delete_snapshot() { |
|
68 |
local snap_id=${1} |
|
69 |
debug 2 "deleting ${snap_id}"
|
|
70 |
xc2 delete-snapshot "${REGARGS[@]}" "${snap_id}" > ${2:-/dev/null} || |
|
71 |
{ error "failed to delete ${snap_id}"; return 1; } |
|
72 |
}
|
|
73 |
||
74 |
istrue() { |
|
75 |
[ -n "$1" -a "$1" != "0" ] |
|
76 |
}
|
|
77 |
isfalse() { |
|
78 |
[ -z "$1" -o "$1" = "0" ] |
|
79 |
}
|
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
80 |
cleanup() { |
|
585
by Ben Howard
longer timeout on volume detachment |
81 |
isfalse ${VOLUME_ATTACHED} || detach_vol "${VOLUME_ID}" |
|
584
by Ben Howard
Due to ec2 failures, wrap everything in retry for EBS publication |
82 |
[ -z "${VOLUME_ID}" ] || retry 3 30 delete_vol "${VOLUME_ID}" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
83 |
[ -z "${TEMP_D}" ] || rm -Rf "${TEMP_D}" |
84 |
}
|
|
85 |
||
|
411
by Scott Moser
better handle cleaning up of instances in failed publish-build-ebs |
86 |
killed() { |
87 |
error "${0##*/} killed"
|
|
88 |
exit 143 # 143 is exit code of: bash -c 'sleep 3& kill $$'; echo $? |
|
89 |
}
|
|
90 |
||
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
91 |
di_field() { |
92 |
local field=$1 f="" |
|
|
556.1.1
by Ben Howard
Inital work |
93 |
local fields=( itype iid ami host host_int state |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
94 |
key_name ami_launch_index product_codes |
95 |
instance_type launch_time placement kernel |
|
96 |
ramdisk )
|
|
97 |
||
98 |
for((i=0;i<${#fields[@]};i++)); do |
|
99 |
[ "${fields[$i]}" = "${field}" ] && { f=$(($i+1)) ; break; } |
|
100 |
done
|
|
101 |
[ -n "$f" ] || return 1 |
|
102 |
local cmd='$1 == "INSTANCE" { print $'$f' }' |
|
103 |
_RET=$(awk '-F\t' "$cmd") |
|
104 |
}
|
|
105 |
||
|
116
by Scott Moser
add yet another layer of indirection on ssh |
106 |
xssh() { |
107 |
local host="$1" |
|
108 |
shift;
|
|
|
120
by Scott Moser
get rid of the ssh config in ec2-image2ebs. |
109 |
debug 2 "${SSH} ${SSH_OPTS} ${SSH_USER:+${SSH_USER}@}$host $*"
|
110 |
${SSH} ${SSH_OPTS} ${SSH_USER:+${SSH_USER}@}$host "$@" |
|
|
116
by Scott Moser
add yet another layer of indirection on ssh |
111 |
}
|
112 |
||
|
287
by Scott Moser
ec2-image2ebs: retry the detach volume and delete volume commands |
113 |
# retry(max,sleeptime, cmd)
|
114 |
# retry cmd up to max times until it suceeds, sleeping sleeptime in between
|
|
115 |
retry() { |
|
116 |
local max=$1 sleep=$2 i=0 ret=0; |
|
117 |
shift 2 || { debug 1 "bad input to retry"; return 1; } |
|
118 |
while :; do |
|
119 |
i=$(($i+1)) |
|
120 |
"$@" ; ret=$? |
|
121 |
[ $ret -eq 0 ] && break |
|
122 |
[ $i -eq 1 ] && debug 1 "cmd failed [$i/$max]: $*" || |
|
123 |
debug 2 "cmd failed [$i/$max]: $*"
|
|
124 |
[ $i -lt $max ] || break |
|
125 |
sleep "$sleep" |
|
126 |
done
|
|
127 |
if [ $ret -eq 0 ]; then |
|
128 |
[ $i -ne 1 ] && debug 1 "cmd passed [$i/$max]: $*" |
|
129 |
return 0
|
|
130 |
fi
|
|
131 |
return $ret |
|
132 |
}
|
|
133 |
||
|
561
by Ben Howard
Make discard default mount option for SSD devices |
134 |
write_discard() { |
135 |
cat <<"EOF"
|
|
136 |
discard() { |
|
|
602
by Ben Howard
Fix for SSD and Lucid |
137 |
local mnt="${1}" |
138 |
sudo sed -i -e 's|defaults|defaults,discard|g' "${mnt}/etc/fstab" |
|
|
561
by Ben Howard
Make discard default mount option for SSD devices |
139 |
}
|
140 |
EOF |
|
141 |
}
|
|
142 |
||
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
143 |
write_hvmify() { |
144 |
cat <<"EOF"
|
|
145 |
hvmify() { |
|
146 |
local mnt="${1}" vol="${2}" |
|
|
259
by Scott Moser
fix a syntax error in ec2-image2ebs, and logic error in publish-build-ebs |
147 |
error "hvmify on ${mnt} for vol=${vol}"
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
148 |
sudo sed -i \
|
149 |
-e 's,^#GRUB_TERMINAL=console,GRUB_TERMINAL=console,' \ |
|
150 |
"${mnt}/etc/default/grub"
|
|
|
643
by Robert C Jennings
Fix bash syntax error |
151 |
if [ -f "${mnt}/etc/default/grub.d/50-cloudimg-settings.cfg" ] ; then |
|
644
by Robert C Jennings
Append grub options as root |
152 |
echo GRUB_HIDDEN_TIMEOUT=0.1 | \ |
153 |
sudo tee --append \
|
|
154 |
"${mnt}/etc/default/grub.d/50-cloudimg-settings.cfg" > /dev/null
|
|
|
642
by Robert C Jennings
Remove 10s grub timeout |
155 |
fi
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
156 |
for m in sys proc dev; do sudo mount --bind "/${m}" "${mnt}/${m}"; done |
157 |
sudo chroot "${mnt}" update-grub |
|
158 |
sudo chroot "${mnt}" grub-install "${vol}" |
|
|
601
by Ben Howard
Sync devices and force unmount |
159 |
sync |
160 |
for m in sys proc dev; do sudo umount -f "${mnt}/${m}"; done; |
|
|
612
by Scott Moser
ec2-image2ebs: only create ttyS0.conf from tty1.conf if tty1.conf exists |
161 |
if [ -f "$mnt/etc/init/tty1.conf" ]; then |
162 |
# no upstart in vivid means no upstart jobs
|
|
163 |
sudo cp "${mnt}/etc/init/tty1.conf" "${mnt}/etc/init/ttyS0.conf" |
|
164 |
sudo sed -i 's,tty1,ttyS0,' "${mnt}/etc/init/ttyS0.conf" |
|
165 |
fi
|
|
|
645.1.1
by Jose Vazquez
xenial boots 25s earlier removing unneeded xen-fbfront driver fixes lp bug #1602466 |
166 |
|
167 |
# from xenial boot 25s earlier removing unneeded xen-fbfront driver
|
|
|
645.1.4
by Jose Vazquez
Simplified SUITE computation & recovered prefix |
168 |
SUITE=$(sudo chroot "$mnt" lsb_release -sc) |
|
645.1.2
by Jose Vazquez
Turned if into case as requested |
169 |
case "${SUITE}" in |
170 |
"precise" | "trusty") |
|
171 |
;; # nothing to do here
|
|
|
645.1.3
by Jose Vazquez
Making the fix from xenial and beyond |
172 |
"xenial" | "yakkety" | *) |
|
645.1.4
by Jose Vazquez
Simplified SUITE computation & recovered prefix |
173 |
BLACKLIST_FILE="${mnt}/etc/modprobe.d/blacklist-xen-fbfront.conf" |
|
645.1.1
by Jose Vazquez
xenial boots 25s earlier removing unneeded xen-fbfront driver fixes lp bug #1602466 |
174 |
sudo rm -f "${BLACKLIST_FILE}"
|
175 |
sudo touch "${BLACKLIST_FILE}"
|
|
176 |
echo "# xenial+ boots 25s earlier removing unneeded xen-fbfront" | \ |
|
177 |
sudo tee --append "${BLACKLIST_FILE}" > /dev/null
|
|
178 |
echo | sudo tee --append "${BLACKLIST_FILE}" > /dev/null |
|
179 |
echo "blacklist xen-fbfront" | \ |
|
180 |
sudo tee --append "${BLACKLIST_FILE}" > /dev/null
|
|
|
645.1.2
by Jose Vazquez
Turned if into case as requested |
181 |
;; |
182 |
esac
|
|
|
645.2.3
by Jose Vazquez
Simplified SUITE computation & recovered mnt prefix |
183 |
|
|
645.2.1
by Jose Vazquez
From xenial boot 1s earlier overriding cdrom udev rule, fixes lp bug #1613470 |
184 |
# from xenial boot 1s earlier overriding cdrom udev rule
|
|
645.2.3
by Jose Vazquez
Simplified SUITE computation & recovered mnt prefix |
185 |
SUITE=$(sudo chroot "${mnt}" lsb_release -sc) |
|
645.2.2
by Jose Vazquez
Converting if to a case and applying from xenial and beyond |
186 |
case "${SUITE}" in |
187 |
"precise" | "trusty") |
|
188 |
;; # Nothing to do here
|
|
189 |
"xenial" | "yakkety" | *) |
|
|
645.2.3
by Jose Vazquez
Simplified SUITE computation & recovered mnt prefix |
190 |
sudo ln -s /dev/null "${mnt}/etc/udev/rules.d/60-cdrom_id.rules"
|
|
645.2.2
by Jose Vazquez
Converting if to a case and applying from xenial and beyond |
191 |
;; |
192 |
esac
|
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
193 |
}
|
194 |
EOF |
|
195 |
}
|
|
196 |
||
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
197 |
Usage() { |
198 |
cat <<EOF
|
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
199 |
Usage: ${0##*/} [ options ] instance-id arch imgurl register-name
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
200 |
|
|
556.1.1
by Ben Howard
Inital work |
201 |
Utilize 'instance-id' to
|
202 |
- download 'imgurl',
|
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
203 |
- populate a ebs snapshot with its content.
|
204 |
- register an ebs boot instance for 'arch' as 'register-name'
|
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
205 |
|
206 |
options:
|
|
207 |
--region r use '--region r' in ec2 commands
|
|
208 |
--description d register image with description 'd'
|
|
|
195
by Scott Moser
ec2-image2ebs: add '--snapshot-desc' argument |
209 |
--snapshot-desc d create snapshot with description 'd'
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
210 |
--kernel aki register image with kernel 'aki'
|
211 |
--ramdisk ari register image with ramdisk 'ari'
|
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
212 |
--size s create volume of size 's' (in Gig)
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
213 |
--fstype f put filesystem of type 'f' on volume
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
214 |
--etype t "ebs type" type (hvm or ebs)
|
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
215 |
--address address instance by 'address' rather than looking up
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
216 |
-v|--verbose increase verbosity
|
|
369
by Scott Moser
ec2-image2ebs, run-instance-and-wait: add prefix, make more quiet |
217 |
--prefix P prefix output messages with 'P'
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
218 |
--sriov register with 'sriov=simple' for HVM
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
219 |
EOF
|
220 |
}
|
|
221 |
||
222 |
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; } |
|
223 |
||
224 |
short_opts="hv" |
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
225 |
long_opts="etype:,address:,description:,help,fstype:,image-md5:,kernel:,prefix:,ramdisk:,region:,snapshot-desc:,size:,ssh-id:,verbose,sriov" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
226 |
getopt_out=$(getopt --name "${0##*/}" \ |
227 |
--options "${short_opts}" --long "${long_opts}" -- "$@") && |
|
228 |
eval set -- "${getopt_out}" || |
|
229 |
bad_Usage |
|
230 |
||
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
231 |
address="" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
232 |
description="" |
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
233 |
etype="ebs" |
|
187
by Scott Moser
if --fstype is not passed, default to same fs type as image |
234 |
fstype="" |
|
189
by Scott Moser
correctly get image label through (tested this time), |
235 |
# if fslabel is set to an empty string, then the filesystem label will be
|
236 |
# copied from the image. If non-empty, this label will be used.
|
|
237 |
# if set to 'none' no label will be written
|
|
238 |
fslabel="" |
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
239 |
kernel="" |
240 |
ramdisk="" |
|
241 |
region="" |
|
242 |
ssh_id="" |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
243 |
size="15" |
|
195
by Scott Moser
ec2-image2ebs: add '--snapshot-desc' argument |
244 |
snapshot_desc="" |
|
215
by Scott Moser
ec2-image2ebs: add --image-md5 flag |
245 |
image_md5="" |
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
246 |
sriov=0 |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
247 |
VERBOSITY=0 |
|
369
by Scott Moser
ec2-image2ebs, run-instance-and-wait: add prefix, make more quiet |
248 |
PREFIX="" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
249 |
|
250 |
while [ $# -ne 0 ]; do |
|
251 |
cur=${1}; next=${2}; |
|
252 |
case "$cur" in |
|
253 |
-h|--help) Usage; exit 0;; |
|
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
254 |
--address) address=${next}; shift;; |
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
255 |
--etype) etype=${next}; shift;; |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
256 |
--description) description=${next}; shift;; |
|
215
by Scott Moser
ec2-image2ebs: add --image-md5 flag |
257 |
--image-md5) image_md5=${next}; shift;; |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
258 |
--kernel) kernel=${next}; shift;; |
259 |
--ramdisk) ramdisk=${next}; shift;; |
|
260 |
--size) size=${next}; shift;; |
|
261 |
--fstype) fstype=${next}; shift;; |
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
262 |
--sriov) sriov=1;; |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
263 |
-v|--verbose)
|
264 |
VERBOSITY=$((${VERBOSITY}+1));; |
|
265 |
--region)
|
|
266 |
region=${next}; shift;; |
|
267 |
--ssh-id)
|
|
268 |
ssh_id="${next}"; shift;; |
|
|
195
by Scott Moser
ec2-image2ebs: add '--snapshot-desc' argument |
269 |
--snapshot-desc)
|
270 |
snapshot_desc=${next}; shift;; |
|
|
378
by Scott Moser
ec2-image2ebs: actually honor --prefix arg |
271 |
--prefix)
|
272 |
PREFIX="${next}"; shift;; |
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
273 |
--) shift; break;; |
274 |
esac
|
|
275 |
shift; |
|
276 |
done
|
|
277 |
||
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
278 |
[ $# -eq 4 ] || |
279 |
bad_Usage "must provide instance-id arch imgurl register-name"
|
|
280 |
||
|
556.1.1
by Ben Howard
Inital work |
281 |
REGARGS=( ); |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
282 |
[ -n "${region}" ] && REGARGS=( "--region" "$region" ) |
283 |
||
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
284 |
iid=${1} |
|
370
by Scott Moser
ec2-image2ebs: retry ssh verify connection. more quiet apt-get. |
285 |
IID=$iid |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
286 |
arch=${2} |
287 |
imgurl=${3} |
|
288 |
register_name=${4} |
|
289 |
||
|
195
by Scott Moser
ec2-image2ebs: add '--snapshot-desc' argument |
290 |
[ -z "${snapshot_desc}" ] && snapshot_desc="${register_name}" |
291 |
||
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
292 |
case "${arch}" in |
293 |
i386) ec2_arch=$arch;; |
|
|
189
by Scott Moser
correctly get image label through (tested this time), |
294 |
amd64|x86_64) ec2_arch=x86_64;; |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
295 |
*) fail "arch ${arch} not supported (i386 or x86_64)";; |
296 |
esac
|
|
|
649.2.19
by Phil Roche
Map legacy arch types to CIM |
297 |
[ "${arch}" = "amd64" ] && arch=x86_64 && debarch=amd64 || debarch=${arch} |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
298 |
|
|
133
by Scott Moser
TMPDIR is the variable to define temp dir, not TEMPDIR |
299 |
TEMP_D=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX) || |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
300 |
fail "failed to make tmp dir"
|
|
411
by Scott Moser
better handle cleaning up of instances in failed publish-build-ebs |
301 |
trap cleanup EXIT
|
302 |
trap killed SIGINT SIGTERM
|
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
303 |
|
|
367
by Scott Moser
utilize 'xc2 ximages' to avoid expensive 'xc2 describe-images' |
304 |
export XC2_XIMAGES_CACHE_D=${XC2_XIMAGES_CACHE_D:-"$TEMP_D/ximgcache"} |
305 |
||
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
306 |
rem_workd="/mnt/${0##*/}/${register_name##*/}.${arch}.${etype}" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
307 |
rem_src_mnt="${rem_workd}/src_d" |
308 |
rem_trg_mnt="${rem_workd}/trg_d" |
|
309 |
rem_dl_d="${rem_workd}/dl" |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
310 |
rem_funcs="${rem_workd}/sh_funcs" |
311 |
desc_instance="${TEMP_D}/instance.info" |
|
312 |
vol_info="${TEMP_D}/volume.info" |
|
313 |
vol_attach_info="${TEMP_D}/vol_attach.info" |
|
314 |
vol_detach_info="${TEMP_D}/vol_detach.info" |
|
315 |
vol_delete_info="${TEMP_D}/vol_delete.info" |
|
316 |
create_snapshot_info="${TEMP_D}/create_snapshot.info" |
|
317 |
desc_snapshot_poll="${TEMP_D}/desc_snapshot.poll" |
|
318 |
register_info="${TEMP_D}/register.info" |
|
319 |
vol_attach_dev=/dev/sdi |
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
320 |
alt_attach_dev=/dev/xvdi |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
321 |
|
|
584
by Ben Howard
Due to ec2 failures, wrap everything in retry for EBS publication |
322 |
retry 3 5 xc2 info-instances "${REGARGS[@]}" "${iid}" > "${desc_instance}" || |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
323 |
fail "failed to get information about instance $iid"
|
324 |
||
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
325 |
if [ -z "$address" ]; then |
326 |
di_field host < "${desc_instance}" && remaddr=${_RET} && |
|
327 |
[ -n "${remaddr}" ] || fail "failed to read hostname for ${iid}" |
|
328 |
else
|
|
329 |
remaddr="$address" |
|
330 |
fi
|
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
331 |
|
332 |
di_field placement < "${desc_instance}" && placement=${_RET} && |
|
333 |
[ -n "${placement}" ] || fail "unable to determine availability zone" |
|
334 |
||
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
335 |
debug 2 "${iid} is at ${remaddr} in zone ${placement}"
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
336 |
|
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
337 |
debug 1 "verifying connection to ${remaddr}"
|
|
471
by Ben Howard
Increased timeouts due to failures in us-west-1 |
338 |
retry 15 30s xssh "${remaddr}" /bin/true || |
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
339 |
fail "failed to verify connection to ${remaddr}"
|
|
116
by Scott Moser
add yet another layer of indirection on ssh |
340 |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
341 |
# [host] attach a volume to the instance, wait for it to become available
|
|
584
by Ben Howard
Due to ec2 failures, wrap everything in retry for EBS publication |
342 |
retry 3 5 xc2 create-volume "${REGARGS[@]}" --size "$size" \ |
|
115
by Scott Moser
some '--region' argument fixes, use $SSH rather than ssh |
343 |
--availability-zone "${placement}" > "${vol_info}" && |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
344 |
vol_id=$(awk '-F\t' '{print $2}' < "${vol_info}") && [ -n "${vol_id}" ] || |
345 |
fail "failed to create a volume"
|
|
346 |
||
347 |
VOLUME_ID=${vol_id} |
|
348 |
||
349 |
debug 2 "created volume ${vol_id}"
|
|
350 |
||
|
582
by Ben Howard
Retry on attach due to us-west-1 failures |
351 |
retry 15 30 xc2 attach-volume "${REGARGS[@]}" --device "${vol_attach_dev}" \ |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
352 |
--instance "${iid}" "${vol_id}" > "${vol_attach_info}" && |
353 |
VOLUME_ATTACHED=1 || |
|
|
586.1.1
by Robert Jennings
Log volume state for attach/detach failures |
354 |
(cat ${vol_attach_info} && |
355 |
get_volstate "$vol" && |
|
356 |
error "volume state is $_RET" && |
|
357 |
fail "failed to attach volume to ${vol_attach_dev}") |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
358 |
|
359 |
debug 2 "volume ${vol_id} attached to ${iid} at ${vol_attach_dev}"
|
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
360 |
|
|
297
by Scott Moser
rename uec -> cloud or cloudimg |
361 |
# [instance] download the published tarfile from http://cloud-images.ubuntu.com
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
362 |
# [instance] extract image, mount loopback
|
|
218
by Scott Moser
ec2-image2ebs: test getter=lftp |
363 |
getter="lftp" |
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
364 |
debug 2 "obtaining image on ${remaddr}"
|
365 |
xssh "${remaddr}" bash -e <<EOF || fail "failed to download/setup image" |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
366 |
exec 1>&2
|
|
564
by Ben Howard
Further cleanup for EBS workers running out of space |
367 |
[ -e "${rem_workd}" ] && sudo rm -rf "${rem_workd}"
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
368 |
[ -e "${rem_dl_d}" ] && sudo rm -rf "${rem_dl_d}"
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
369 |
sudo mkdir -p "${rem_src_mnt}" "${rem_trg_mnt}" "${rem_dl_d}"
|
|
556.1.1
by Ben Howard
Inital work |
370 |
sudo chown -R \$(id -u) "${rem_workd}"
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
371 |
cat > "${rem_funcs}" <<"END"
|
|
369
by Scott Moser
ec2-image2ebs, run-instance-and-wait: add prefix, make more quiet |
372 |
error() { echo "$PREFIX\$@" 1>&2; }
|
|
147
by Scott Moser
ec2-image2ebs: improve debug/error on no attached device |
373 |
fail() { [ \$# -eq 0 ] || error "\$@" 1>&2; exit 1; }
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
374 |
[ ${VERBOSITY:-0} -ge 2 ] || exec 2>/dev/null
|
375 |
[ ${VERBOSITY:-0} -ge 3 ] && set -x
|
|
376 |
exec 1>&2
|
|
377 |
END
|
|
|
594
by Ben Howard
Use Google DNS resolvers |
378 |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
379 |
. ${rem_funcs}
|
380 |
||
|
214
by Scott Moser
add support for getting with lftp (but disabled at the moment) |
381 |
out="${rem_dl_d}/img.tar.gz"
|
382 |
if [ ! -f "\${out}" ]; then
|
|
|
222
by Scott Moser
add download speed message |
383 |
start=\${SECONDS}
|
|
214
by Scott Moser
add support for getting with lftp (but disabled at the moment) |
384 |
case "$getter" in
|
385 |
lftp)
|
|
|
459
by Scott Moser
improve output of in-instance downloading code |
386 |
dpkg-query --show lftp >/dev/null 2>&1 ||
|
|
375
by Scott Moser
redirect apt stdout to /dev/null for more quiet |
387 |
sudo env DEBIAN_FRONTEND=noninteractive \
|
388 |
apt-get install lftp -qq >/dev/null
|
|
|
214
by Scott Moser
add support for getting with lftp (but disabled at the moment) |
389 |
imgurl="${imgurl}"
|
390 |
proto=\${imgurl%%://*}
|
|
391 |
site=\${imgurl#\${proto}://}; site=\${site%%/*};
|
|
392 |
path=\${imgurl#\${proto}://\${site}/}
|
|
|
222
by Scott Moser
add download speed message |
393 |
lftp -c "open \${proto}://\${site}/ && pget \${path} -o \${out}" </dev/null
|
|
214
by Scott Moser
add support for getting with lftp (but disabled at the moment) |
394 |
;;
|
395 |
*)
|
|
396 |
wget "${imgurl}" --progress=dot:mega -O "\${out}"
|
|
397 |
;;
|
|
398 |
esac
|
|
|
222
by Scott Moser
add download speed message |
399 |
elapsed=\$((\${SECONDS}-\${start}))
|
400 |
[ "\${elapsed}" = "0" ] && elapsed=1
|
|
401 |
size=\$(stat --format "%s" "\${out}")
|
|
|
225
by Scott Moser
ec2-image2ebs: bash typo fix |
402 |
rate=\$(((\${size}/1024)/\${elapsed}))
|
|
459
by Scott Moser
improve output of in-instance downloading code |
403 |
error "got \${size} bytes in \${elapsed} seconds. \${rate} kB/s"
|
|
214
by Scott Moser
add support for getting with lftp (but disabled at the moment) |
404 |
fi
|
405 |
||
|
215
by Scott Moser
ec2-image2ebs: add --image-md5 flag |
406 |
if [ -n "${image_md5}" ]; then
|
407 |
printf "%s %s" "${image_md5}" "img.tar.gz" > "${rem_dl_d}/MD5SUMS"
|
|
|
369
by Scott Moser
ec2-image2ebs, run-instance-and-wait: add prefix, make more quiet |
408 |
( cd "${rem_dl_d}" && md5sum --check MD5SUMS ) >/dev/null
|
|
215
by Scott Moser
ec2-image2ebs: add --image-md5 flag |
409 |
fi
|
410 |
||
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
411 |
tar -C "${rem_dl_d}" -Sxzf "${rem_dl_d}/img.tar.gz"
|
|
562
by Ben Howard
Remove img.tar.gz when done with untaring due to disk space errors |
412 |
sudo rm "${rem_dl_d}/img.tar.gz"
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
413 |
imgfile=""
|
414 |
for f in "${rem_dl_d}/"*; do
|
|
415 |
case \$f in
|
|
416 |
*.img) imgfile=\$f; break;;
|
|
417 |
esac
|
|
418 |
done
|
|
419 |
[ -n "\$imgfile" ] || fail "failed to find *.img in tarball"
|
|
|
188
by Scott Moser
fix typo/bug preventing use of e2label |
420 |
img_label=\$(e2label "\${imgfile}")
|
|
187
by Scott Moser
if --fstype is not passed, default to same fs type as image |
421 |
echo "img_label=\"\${img_label}\"" >> "${rem_funcs}"
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
422 |
sudo mount -o loop \${imgfile} "${rem_src_mnt}"
|
|
187
by Scott Moser
if --fstype is not passed, default to same fs type as image |
423 |
|
424 |
img_fstype=\$(awk '\$2 == "${rem_src_mnt}" { val=\$3 }; END { print val }' < /proc/mounts)
|
|
425 |
echo "img_fstype=\"\${img_fstype}\"" >> "${rem_funcs}"
|
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
426 |
EOF
|
427 |
||
428 |
# [instance] mkfs on attached volume
|
|
429 |
# [instance] copy data from loopbacked image to ebs volume, sync
|
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
430 |
|
|
602
by Ben Howard
Fix for SSD and Lucid |
431 |
if [[ "${etype}" =~ ssd ]]; then |
|
561
by Ben Howard
Make discard default mount option for SSD devices |
432 |
debug 2 "writing discard to rem_funcs ${rem_funcs}" |
433 |
write_discard | xssh "$remaddr" sh -c "cat >> ${rem_funcs}" |
|
434 |
fi
|
|
435 |
||
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
436 |
if [[ "${etype}" =~ hvm ]]; then |
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
437 |
debug 2 "writing hvmify to rem_funcs ${rem_funcs}" |
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
438 |
write_hvmify | xssh "$remaddr" sh -c "cat >> ${rem_funcs}" |
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
439 |
fi
|
440 |
||
|
322
by Scott Moser
hopefully fix random ssh-keyscan timeouts in run-instance-and-wait |
441 |
debug 2 "copying data on ${remaddr}"
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
442 |
xssh "$remaddr" bash -e <<EOF || fail "failed to copy data to volume" |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
443 |
. ${rem_funcs}
|
444 |
i=0;
|
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
445 |
|
446 |
vol_attach_dev=${vol_attach_dev};
|
|
447 |
alt_attach_dev="${alt_attach_dev}";
|
|
448 |
||
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
449 |
while [ \$i -lt 180 ]; do
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
450 |
[ -e "${vol_attach_dev}" ] && break ||
|
451 |
{ [ -e "\${alt_attach_dev}" ] &&
|
|
452 |
vol_attach_dev=\${alt_attach_dev} && break; } ||
|
|
453 |
sleep 2
|
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
454 |
i=\$((\$i+1))
|
455 |
done
|
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
456 |
|
457 |
[ -e "\${vol_attach_dev}" ] || {
|
|
458 |
error "\${vol_attach_dev} doesn't exist, showing /proc/partitions"
|
|
|
147
by Scott Moser
ec2-image2ebs: improve debug/error on no attached device |
459 |
cat /proc/partitions 1>&2
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
460 |
fail "\${vol_attach_dev} does not exist"
|
|
147
by Scott Moser
ec2-image2ebs: improve debug/error on no attached device |
461 |
}
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
462 |
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
463 |
fsdev="\${vol_attach_dev}"
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
464 |
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
465 |
if [[ "${etype}" =~ hvm ]]; then
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
466 |
echo "1,,L,*" | sudo sfdisk "\${vol_attach_dev}" ||
|
467 |
fail "failed to partition \${vol_attach_dev}"
|
|
468 |
fsdev="\${vol_attach_dev}1"
|
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
469 |
fi
|
470 |
||
|
186
by Scott Moser
use the image filesystems label as the default for the ebs root fs label |
471 |
label=\${img_label}
|
472 |
if [ -n "${fslabel}" ]; then
|
|
473 |
[ "${fslabel}" = "none" ] && fslabel=""
|
|
474 |
label="${fslabel}"
|
|
475 |
fi
|
|
|
189
by Scott Moser
correctly get image label through (tested this time), |
476 |
xtype=\${img_fstype}
|
|
187
by Scott Moser
if --fstype is not passed, default to same fs type as image |
477 |
if [ -n "${fstype}" ]; then
|
478 |
xtype="${fstype}"
|
|
479 |
fi
|
|
480 |
mkfs_args="-t \${xtype}"
|
|
481 |
case "\${xtype}" in
|
|
482 |
ext*) mkfs_args="\${mkfs_args} -F";;
|
|
483 |
esac
|
|
484 |
||
|
369
by Scott Moser
ec2-image2ebs, run-instance-and-wait: add prefix, make more quiet |
485 |
error "creating fs: \$mkfs_args label=\${label} on \$fsdev"
|
486 |
sudo mkfs \${mkfs_args} \${label:+-L "\${label}"} "\${fsdev}" >/dev/null
|
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
487 |
sudo mount "\${fsdev}" "${rem_trg_mnt}"
|
|
114
by Scott Moser
add flags to rsync to preserve more |
488 |
sudo rsync -aXHAS "${rem_src_mnt}/" "${rem_trg_mnt}"
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
489 |
|
|
602
by Ben Howard
Fix for SSD and Lucid |
490 |
suite=\$(sudo chroot ${rem_trg_mnt} lsb_release --codename --short)
|
|
610
by Ben Howard
Fix detection of SSD volumes |
491 |
if [[ ! "\${suite}" =~ (lucid|10.04) ]]; then
|
492 |
# Only enable discard for Lucid and later
|
|
493 |
||
|
602
by Ben Howard
Fix for SSD and Lucid |
494 |
if [[ "${etype}" =~ ssd ]]; then
|
495 |
discard "${rem_trg_mnt}"
|
|
496 |
fi
|
|
|
561
by Ben Howard
Make discard default mount option for SSD devices |
497 |
|
|
602
by Ben Howard
Fix for SSD and Lucid |
498 |
if [[ "${etype}" =~ hvm ]]; then
|
499 |
hvmify "${rem_trg_mnt}" "\${vol_attach_dev}"
|
|
500 |
fi
|
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
501 |
fi
|
502 |
||
|
587
by Ben Howard
Be aggressive about calling detach on volumes |
503 |
sudo umount -f "${rem_trg_mnt}"
|
504 |
sudo umount -f "${rem_src_mnt}"
|
|
505 |
sync
|
|
|
563
by Ben Howard
Remove source locations for ebs worker reuse |
506 |
sudo rm -rf "${rem_src_mnt}"
|
|
564
by Ben Howard
Further cleanup for EBS workers running out of space |
507 |
sudo rm -rf "${rem_workd}"
|
|
287
by Scott Moser
ec2-image2ebs: retry the detach volume and delete volume commands |
508 |
|
|
490
by Ben Howard
Enabled the use of 12.04 as a utility AMI for ebs creation |
509 |
error "copied data to \${vol_attach_dev}"
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
510 |
EOF
|
511 |
||
512 |
# [host] detach volume, create-snapshot, delete-volume
|
|
|
585
by Ben Howard
longer timeout on volume detachment |
513 |
detach_vol "${vol_id}" "${vol_detach_info}" || |
|
586.1.1
by Robert Jennings
Log volume state for attach/detach failures |
514 |
(get_volstate "$vol" && |
515 |
error "volume state is $_RET" && |
|
516 |
fail "failed to detach volume") |
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
517 |
|
518 |
VOLUME_ATTACHED=0 |
|
519 |
||
520 |
debug 2 "creating snapshot of ${vol_id}"
|
|
521 |
snap_start=${SECONDS} |
|
|
584
by Ben Howard
Due to ec2 failures, wrap everything in retry for EBS publication |
522 |
retry 10 60 xc2 create-snapshot "${REGARGS[@]}" \ |
|
195
by Scott Moser
ec2-image2ebs: add '--snapshot-desc' argument |
523 |
"--description=${snapshot_desc}" \ |
|
144
by Scott Moser
ec2-image2ebs: pass set description of snapshot to register name |
524 |
"${vol_id}" > "${create_snapshot_info}" && |
|
584
by Ben Howard
Due to ec2 failures, wrap everything in retry for EBS publication |
525 |
snapshot_id=$(awk '-F\t' 'END{print $2}' < "${create_snapshot_info}") && |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
526 |
[ -n "${snapshot_id}" ] || |
527 |
fail "failed to create snapshot of ${vol_id}"
|
|
528 |
SNAPSHOT_ID=${snapshot_id} |
|
529 |
||
|
584
by Ben Howard
Due to ec2 failures, wrap everything in retry for EBS publication |
530 |
retry 5 60 delete_vol "${vol_id}" "${vol_delete_info}" || |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
531 |
fail "failed to delete volume ${vol_id}"
|
532 |
VOLUME_ID="" |
|
533 |
||
534 |
failed_last=0 |
|
535 |
||
536 |
debug 2 "waiting for ${snapshot_id} to become complete"
|
|
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
537 |
# [host] wait for snapshot to leave pending
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
538 |
for((i=0;i<90;i++)); do |
|
584
by Ben Howard
Due to ec2 failures, wrap everything in retry for EBS publication |
539 |
retry 3 10 xc2 info-snapshots "${REGARGS[@]}" "${snapshot_id}" \ |
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
540 |
> "${desc_snapshot_poll}" && failed_last=0 || |
541 |
failed_last=$((${failed_last}+1)) |
|
542 |
[ "${failed_last}" -lt 30 ] || |
|
543 |
fail "describe-snapshots failed ${failed_last} times in a row"
|
|
544 |
if [ ${failed_last} -eq 0 ]; then |
|
545 |
state=$(awk '-F\t' '{print $4}' < "${desc_snapshot_poll}") |
|
546 |
case "${state}" in |
|
547 |
pending) :;;
|
|
548 |
completed) break;; |
|
549 |
error) fail "snapshot ${snapshot_id} is in state 'errror'";; |
|
550 |
*) error "WARNING: unknown snapshot state \"${state}\"";; |
|
551 |
esac
|
|
552 |
debug 2 "snapshot state is $state after $((${SECONDS}-${snap_start}))s" |
|
553 |
fi
|
|
|
471
by Ben Howard
Increased timeouts due to failures in us-west-1 |
554 |
sleep 15
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
555 |
done
|
556 |
||
557 |
debug 1 "snapshot complete after $((${SECONDS}-${snap_start})) seconds"
|
|
558 |
||
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
559 |
# [host] register
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
560 |
args=( "${REGARGS[@]}" "--architecture=${ec2_arch}" "--name=${register_name}" ) |
561 |
[ -n "${description}" ] && args[${#args[@]}]="--description=${description}" |
|
562 |
[ -n "${kernel}" ] && args[${#args[@]}]="--kernel=${kernel}" |
|
563 |
[ -n "${ramdisk}" ] && args[${#args[@]}]="--ramdisk=${ramdisk}" |
|
564 |
||
|
239
by Scott Moser
ec2-image2ebs: add comment on use of --block-device-mapping in ec2-register |
565 |
# Note (2010-08-31):
|
566 |
# passing --block-device-mapping as shown here does get block devices
|
|
567 |
# mapped for the 2 explicit cases. Other block devices will not be
|
|
568 |
# present by default, though. ie, on an m1.large, which should have 2
|
|
569 |
# ephemeral block devices, you will get only 1.
|
|
570 |
#
|
|
571 |
# if you do not pass *any* --block-device-mapping, you will get no
|
|
572 |
# ephemeral block devices.
|
|
573 |
#
|
|
574 |
# if you pass a long list:
|
|
575 |
# --block-device-mapping sdb=ephemeral0
|
|
576 |
# --block-device-mapping sdc=ephemeral1
|
|
577 |
# --block-device-mapping sdd=ephemeral2
|
|
578 |
# --block-device-mapping sde=ephemeral3
|
|
579 |
#
|
|
580 |
# then you will get that all appropriate block devices for the given size
|
|
581 |
# when you run it (ie, m1.large will have sdb and sdc), but the
|
|
582 |
# metadata will also have entries for sdd and sde. That fact will *not*
|
|
583 |
# prevent you from attaching a volume as '--device /dev/sde'
|
|
584 |
#
|
|
585 |
# the end result of all of this is that almost all ubuntu EBS instances
|
|
586 |
# will only have a single ephemeral device node unless the user
|
|
587 |
# changes arguments at launch time.
|
|
|
258
by Scott Moser
initial commit for publishing to hvm instances (cluster compute) |
588 |
case "${ec2_arch}:${etype}" in |
|
556.1.1
by Ben Howard
Inital work |
589 |
x86_64:ebs*)
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
590 |
args[${#args[@]}]="--block-device-mapping" |
591 |
args[${#args[@]}]="/dev/sdb=ephemeral0" |
|
592 |
;; |
|
|
556.1.1
by Ben Howard
Inital work |
593 |
i386:ebs*)
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
594 |
args[${#args[@]}]="--block-device-mapping" |
595 |
args[${#args[@]}]="/dev/sda2=ephemeral0" |
|
596 |
;; |
|
|
556.1.1
by Ben Howard
Inital work |
597 |
x86_64:hvm*)
|
|
263
by Scott Moser
add appropriate block-device-mappings for hvm types |
598 |
args[${#args[@]}]="--block-device-mapping" |
599 |
args[${#args[@]}]="/dev/sdb=ephemeral0" |
|
600 |
args[${#args[@]}]="--block-device-mapping" |
|
601 |
args[${#args[@]}]="/dev/sdc=ephemeral1" |
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
602 |
[ "${sriov:-0}" -eq 1 ] && |
603 |
args[${#args[@]}]="--sriov=simple" |
|
|
556.1.1
by Ben Howard
Inital work |
604 |
;; |
605 |
esac
|
|
606 |
||
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
607 |
# Handle the specific mappings
|
608 |
debug 1 "ec2-images2ebs: etype is ${etype}"
|
|
|
556.1.1
by Ben Howard
Inital work |
609 |
case "${etype}" in |
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
610 |
*ssd)
|
611 |
# SSDs use the gp2 disk type
|
|
|
556.1.1
by Ben Howard
Inital work |
612 |
args[${#args[@]}]="--block-device-mapping" |
613 |
args[${#args[@]}]="/dev/sda1=${snapshot_id}::true:gp2" |
|
614 |
;; |
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
615 |
*io1)
|
|
570
by Ben Howard
Removed provisioned IOPS setting per AWS request |
616 |
# Provisioned IOPs use io1
|
|
556.1.1
by Ben Howard
Inital work |
617 |
args[${#args[@]}]="--block-device-mapping" |
|
572
by Ben Howard
Reduce PIOPS to the proper amount. Since these are 8GB volumes, the max 30:1 ratio is 240 |
618 |
args[${#args[@]}]="/dev/sda1=${snapshot_id}::true:io1:200" |
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
619 |
;; |
620 |
*)
|
|
621 |
args[${#args[@]}]="--snapshot" |
|
|
556.1.1
by Ben Howard
Inital work |
622 |
args[${#args[@]}]="${snapshot_id}" |
623 |
;; |
|
624 |
esac
|
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
625 |
|
|
556.1.2
by Ben Howard
Working code for production of EBS io1 and SSD disks for both paravirtual |
626 |
[[ "${etype}" =~ hvm ]] && args[${#args[@]}]="--virtualization-type=hvm" |
627 |
||
|
650.1.2
by Phil Roche
Adding comments as to why we are using cloud-image-manager |
628 |
# If the storage type contains the string hvm then this is of virtualization type hvm
|
|
650.1.5
by Phil Roche
Update calls to register_image_from_snapshot due to cloud-image-manager refactor |
629 |
# As such we will use cloud-image-manager to register the image and set AMI
|
630 |
# attributes during registration that can not be set using the old way
|
|
|
650.1.2
by Phil Roche
Adding comments as to why we are using cloud-image-manager |
631 |
# to register images.
|
|
649.2.16
by Phil Roche
Added extra debugging infor for HVM-EBS image builds |
632 |
if [[ "${etype}" =~ hvm ]] |
633 |
then
|
|
|
650.1.5
by Phil Roche
Update calls to register_image_from_snapshot due to cloud-image-manager refactor |
634 |
# Map ebs volume type to the terms used by cloud-image-manager
|
|
650.1.4
by Phil Roche
Rename ebs_volume_type to ebs_storage_type to avoid confsion with EBS volume types |
635 |
ebs_storage_type='ebs' |
|
649.2.18
by Phil Roche
Call register_images_from_snapshot from ec2image2ebs |
636 |
if [[ "${etype}" =~ ssd ]]; then |
|
650.1.4
by Phil Roche
Rename ebs_volume_type to ebs_storage_type to avoid confsion with EBS volume types |
637 |
ebs_storage_type='ebs-ssd' |
|
649.2.18
by Phil Roche
Call register_images_from_snapshot from ec2image2ebs |
638 |
fi
|
639 |
if [[ "${etype}" =~ io1 ]]; then |
|
|
650.1.4
by Phil Roche
Rename ebs_volume_type to ebs_storage_type to avoid confsion with EBS volume types |
640 |
ebs_storage_type='ebs-io1' |
|
649.2.18
by Phil Roche
Call register_images_from_snapshot from ec2image2ebs |
641 |
fi
|
642 |
||
|
651.2.1
by Phil Roche
Registration or release image was using the incorrect label for query data |
643 |
query_data_directory="${SUITE}/$SERIAL/${RELEASE_LABEL}/ec2-query-data/${debarch}/hvm/ebs/${ebs_storage_type}" |
|
649.2.18
by Phil Roche
Call register_images_from_snapshot from ec2image2ebs |
644 |
mkdir -p "${query_data_directory}"
|
645 |
||
|
651.2.1
by Phil Roche
Registration or release image was using the incorrect label for query data |
646 |
if [ ${RELEASE_LABEL} = 'release' ]; then |
647 |
query_data_file=${query_data_directory}/released.txt |
|
648 |
else
|
|
649 |
query_data_file=${query_data_directory}/${RELEASE_LABEL}.txt |
|
650 |
fi
|
|
651 |
||
|
650.1.5
by Phil Roche
Update calls to register_image_from_snapshot due to cloud-image-manager refactor |
652 |
debug 1 "Running cloud-image-manager register_image_from_snapshot ..." |
|
657.1.3
by Daniel Watkins
Log the full cloud-image-manager command used for EBS images |
653 |
set -x
|
|
649.2.18
by Phil Roche
Call register_images_from_snapshot from ec2image2ebs |
654 |
${VENV_BIN}/cloud-image-manager \ |
|
650.1.2
by Phil Roche
Adding comments as to why we are using cloud-image-manager |
655 |
--debug-log debug.log \
|
656 |
--log-to-console \
|
|
657 |
aws \
|
|
|
657.1.4
by Daniel Watkins
Allow AWS profile/partition passed to cloud-image-manager to be modified for EBS images |
658 |
--partition "${AWS_DEFAULT_PARTITION:-aws}" \ |
659 |
--profile "${AWS_DEFAULT_PROFILE:-alan}" \ |
|
|
650.1.5
by Phil Roche
Update calls to register_image_from_snapshot due to cloud-image-manager refactor |
660 |
register_image_from_snapshot \
|
|
650.1.2
by Phil Roche
Adding comments as to why we are using cloud-image-manager |
661 |
"${SUITE}" \ |
662 |
"${SERIAL}" \ |
|
|
651.2.1
by Phil Roche
Registration or release image was using the incorrect label for query data |
663 |
"${RELEASE_LABEL}" \ |
|
650.1.2
by Phil Roche
Adding comments as to why we are using cloud-image-manager |
664 |
--arch=${debarch} \ |
665 |
--virt="hvm" \ |
|
666 |
--snapshot_id=${SNAPSHOT_ID} \ |
|
|
650.1.4
by Phil Roche
Rename ebs_volume_type to ebs_storage_type to avoid confsion with EBS volume types |
667 |
--ebs_storage_type=${ebs_storage_type} \ |
|
650.1.2
by Phil Roche
Adding comments as to why we are using cloud-image-manager |
668 |
--region ${region} \ |
669 |
--query-data-directory=${query_data_directory} \ |
|
|
655.1.1
by Phil Roche
cloud-image-manager image registrations should not make public |
670 |
--force-querydata \
|
|
650.1.8
by Phil Roche
Debug information for register_image_from_snapshot is output by default |
671 |
--no-debug |
|
657.1.3
by Daniel Watkins
Log the full cloud-image-manager command used for EBS images |
672 |
set +x
|
|
649.2.18
by Phil Roche
Call register_images_from_snapshot from ec2image2ebs |
673 |
|
|
649.2.21
by Phil Roche
Read query data from CIM registered HVM EBS image |
674 |
# read the img_id from the generated query data
|
|
651.2.1
by Phil Roche
Registration or release image was using the incorrect label for query data |
675 |
read junk junk junk junk junk junk junk new_ami junk < ${query_data_file} |
|
649.2.21
by Phil Roche
Read query data from CIM registered HVM EBS image |
676 |
|
677 |
echo "${new_ami}" > "${register_info}" |
|
|
649.2.16
by Phil Roche
Added extra debugging infor for HVM-EBS image builds |
678 |
else
|
679 |
debug 1 "registering: register ${args[*]}" |
|
680 |
retry 15 30 xc2 ximages register "${args[@]}" > "${register_info}" && |
|
681 |
new_ami=$(awk '-F\t' 'END{print $2}' < "${register_info}") && |
|
682 |
[ -n "${new_ami}" ] && SNAPSHOT_ID="" || |
|
683 |
fail "failed to register: xc2 register ${args[*]}"
|
|
684 |
fi
|
|
685 |
||
|
146
by Scott Moser
ec2-image2ebs: add debug statement with new_ami in it |
686 |
debug 1 "published ${register_name} as ${new_ami}"
|
|
113
by Scott Moser
first generally functional ec2-image2ebs script |
687 |
echo "${new_ami}" |
|
112
by Scott Moser
add non-working version of ec2-image2ebs |
688 |
exit 0
|