~josvaz/ubuntu-on-ec2/ec2-publishing-scripts-lp1613470-udev-cdrom

#!/bin/bash

# vi: ts=4 noexpandtab

TEMP_D=""

VOLUME_ID=""

VOLUME_ATTACHED=0

SNAPSHOT_ID=""

SSH_USER=${SSH_USER:-ubuntu}

REGARGS=( )

IID=""

SSH=${SSH:-"ssh"}

error() { echo ${PREFIX:+"${PREFIX}"} "$@" 1>&2; }

fail() {

	isfalse ${SNAPSHOT_ID} || delete_snapshot "${SNAPSHOT_ID}"

	[ $# -eq 0 ] || error "$@";

	local out=""

	[ -z "$IID" ] || out=$(xc2 info-instances "${REGARGS[@]}" "${IID}")

	[ -n "$out" ] || error "$out"

	exit 1;

}

debug() {

	local level=${1}

	shift;

	[ "${level}" -gt "${VERBOSITY}" ] && return

	echo "$(date): ${@}" 1>&2

}

get_volstate() {

	local vol="$1" tmp=""

	tmp=$(mktemp "$TEMP_D/describe_vol_out.$vol.XXXXXX") ||

		return 1

	if xc2 describe-volumes "${REGARGS[@]}" "$vol" > "$tmp"; then

		_RET=$(awk '-F\t' '$1 == "VOLUME" { print $6 }' "$tmp")

		rm -f "$tmp"

		return 0

	fi

	error "ec2-describe-volumes $vol failed"

	rm -f "$tmp"

	_RET=""

}

detach_vol() {

	local vol="${1}" log=${2:-/dev/null} max=${3:-20} state="" i=0 ret=1;

	debug 2 "detaching ${vol}"

	until [ "${i}" -gt "${max}" -o "$state" = "available" ]; do

		i=$(($i + 1))

		retry 3 30 get_volstate "$vol" && state="${_RET}"

		debug 2 "volume state after pass $i/$max, state=$state"

		case "${state}" in

			available) debug 2 "${vol} is available, returning 0"; ret=0;;

			in-use)	   debug 2 "${vol} is attached calling for detach";

					   retry 3 30 xc2 detach-volume "${REGARGS[@]}" "${vol}" > ${log};

					   sleep 60;;

		esac

	done

	error "after $i waits, $vol, state=$state"

	return $ret

}

delete_vol() {

	local vol=${1}

	debug 2 "deleting ${vol}"

	xc2 delete-volume "${REGARGS[@]}" "${vol}" > ${2:-/dev/null} ||

		{ error "failed to delete ${vol}"; return 1; }

}

delete_snapshot() {

	local snap_id=${1}

	debug 2 "deleting ${snap_id}"

	xc2 delete-snapshot "${REGARGS[@]}" "${snap_id}" > ${2:-/dev/null} ||

		{ error "failed to delete ${snap_id}"; return 1; }

}

istrue() {

	[ -n "$1" -a "$1" != "0" ]

}

isfalse() {

	[ -z "$1" -o "$1" = "0" ]

}

cleanup() {

	isfalse ${VOLUME_ATTACHED} || detach_vol "${VOLUME_ID}"

	[ -z "${VOLUME_ID}" ] || retry 3 30 delete_vol "${VOLUME_ID}"

	[ -z "${TEMP_D}" ] || rm -Rf "${TEMP_D}"

}

killed() {

	error "${0##*/} killed"

	exit 143 # 143 is exit code of: bash -c 'sleep 3& kill $$'; echo $?

}

di_field() {

	local field=$1 f=""

	local fields=( itype iid ami host host_int state

		key_name ami_launch_index product_codes

		instance_type launch_time placement kernel

		ramdisk )

	for((i=0;i<${#fields[@]};i++)); do

		[ "${fields[$i]}" = "${field}" ] && { f=$(($i+1)) ; break; }

	done

	[ -n "$f" ] || return 1

	local cmd='$1 == "INSTANCE" { print $'$f' }'

	_RET=$(awk '-F\t' "$cmd")

}

xssh() {

    local host="$1"

	shift;

	debug 2 "${SSH} ${SSH_OPTS} ${SSH_USER:+${SSH_USER}@}$host $*"

	${SSH} ${SSH_OPTS} ${SSH_USER:+${SSH_USER}@}$host "$@"

}

# retry(max,sleeptime, cmd)

# retry cmd up to max times until it suceeds, sleeping sleeptime in between

retry() {

    local max=$1 sleep=$2 i=0 ret=0;

    shift 2 || { debug 1 "bad input to retry"; return 1; }

    while :; do

        i=$(($i+1))

        "$@" ; ret=$?

        [ $ret -eq 0 ] && break

        [ $i -eq 1 ] && debug 1 "cmd failed [$i/$max]: $*" ||

            debug 2 "cmd failed [$i/$max]: $*"

        [ $i -lt $max ] || break

        sleep "$sleep"

    done

    if [ $ret -eq 0 ]; then

        [ $i -ne 1 ] && debug 1 "cmd passed [$i/$max]: $*"

        return 0

    fi

    return $ret

}

write_discard() {

cat <<"EOF"

discard() {

	local mnt="${1}"

	sudo sed -i -e 's|defaults|defaults,discard|g' "${mnt}/etc/fstab"

}

EOF

}

write_hvmify() {

cat <<"EOF"

hvmify() {

	local mnt="${1}" vol="${2}"

	error "hvmify on ${mnt} for vol=${vol}"

	sudo sed -i \

		-e 's,^#GRUB_TERMINAL=console,GRUB_TERMINAL=console,' \

		"${mnt}/etc/default/grub"

	if [ -f "${mnt}/etc/default/grub.d/50-cloudimg-settings.cfg" ] ; then

		echo GRUB_HIDDEN_TIMEOUT=0.1 | \

			sudo tee --append \

				"${mnt}/etc/default/grub.d/50-cloudimg-settings.cfg" > /dev/null

	fi

	for m in sys proc dev; do sudo mount --bind "/${m}" "${mnt}/${m}"; done

	sudo chroot "${mnt}" update-grub

	sudo chroot "${mnt}" grub-install "${vol}"

	sync

	for m in sys proc dev; do sudo umount -f "${mnt}/${m}"; done;

	if [ -f "$mnt/etc/init/tty1.conf" ]; then

		# no upstart in vivid means no upstart jobs

		sudo cp "${mnt}/etc/init/tty1.conf" "${mnt}/etc/init/ttyS0.conf"

		sudo sed -i 's,tty1,ttyS0,' "${mnt}/etc/init/ttyS0.conf"

	fi

}

EOF

}

Usage() {

	cat <<EOF

Usage: ${0##*/} [ options ] instance-id arch imgurl register-name

   Utilize 'instance-id' to

     - download 'imgurl',

     - populate a ebs snapshot with its content.

     - register an ebs boot instance for 'arch' as 'register-name'

   options:

       --region r        use '--region r' in ec2 commands

       --description d   register image with description 'd'

       --snapshot-desc d create snapshot with description 'd'

       --kernel aki      register image with kernel 'aki'

       --ramdisk ari     register image with ramdisk 'ari'

       --size s          create volume of size 's' (in Gig)

       --fstype f        put filesystem of type 'f' on volume

       --etype  t        "ebs type" type (hvm or ebs)

       --address         address instance by 'address' rather than looking up

    -v|--verbose         increase verbosity

       --prefix P        prefix output messages with 'P'

       --sriov			 register with 'sriov=simple' for HVM

EOF

}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }

short_opts="hv"

long_opts="etype:,address:,description:,help,fstype:,image-md5:,kernel:,prefix:,ramdisk:,region:,snapshot-desc:,size:,ssh-id:,verbose,sriov"

getopt_out=$(getopt --name "${0##*/}" \

	--options "${short_opts}" --long "${long_opts}" -- "$@") &&

	eval set -- "${getopt_out}" ||

	bad_Usage

address=""

description=""

etype="ebs"

fstype=""

# if fslabel is set to an empty string, then the filesystem label will be

# copied from the image.  If non-empty, this label will be used.

# if set to 'none' no label will be written

fslabel=""

kernel=""

ramdisk=""

region=""

ssh_id=""

size="15"

snapshot_desc=""

image_md5=""

sriov=0

VERBOSITY=0

PREFIX=""

while [ $# -ne 0 ]; do

	cur=${1}; next=${2};

	case "$cur" in

		-h|--help) Usage; exit 0;;

		   --address) address=${next}; shift;;

		   --etype) etype=${next}; shift;;

		   --description) description=${next}; shift;;

		   --image-md5) image_md5=${next}; shift;;

		   --kernel) kernel=${next}; shift;;

		   --ramdisk) ramdisk=${next}; shift;;

		   --size) size=${next}; shift;;

		   --fstype) fstype=${next}; shift;;

		   --sriov) sriov=1;;

		-v|--verbose)

			VERBOSITY=$((${VERBOSITY}+1));;

		   --region)

			region=${next}; shift;;

		   --ssh-id)

			ssh_id="${next}"; shift;;

		   --snapshot-desc)

			snapshot_desc=${next}; shift;;

		   --prefix)

			PREFIX="${next}"; shift;;

		--) shift; break;;

	esac

	shift;

done

[ $# -eq 4 ] ||

	bad_Usage "must provide instance-id arch imgurl register-name"

REGARGS=( );

[ -n "${region}" ] && REGARGS=( "--region" "$region" )

iid=${1}

IID=$iid

arch=${2}

imgurl=${3}

register_name=${4}

[ -z "${snapshot_desc}" ] && snapshot_desc="${register_name}"

case "${arch}" in

	i386) ec2_arch=$arch;;

	amd64|x86_64) ec2_arch=x86_64;;

	*) fail "arch ${arch} not supported (i386 or x86_64)";;

esac

TEMP_D=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX) ||

	fail "failed to make tmp dir"

trap cleanup EXIT

trap killed SIGINT SIGTERM

export XC2_XIMAGES_CACHE_D=${XC2_XIMAGES_CACHE_D:-"$TEMP_D/ximgcache"}

rem_workd="/mnt/${0##*/}/${register_name##*/}.${arch}.${etype}"

rem_src_mnt="${rem_workd}/src_d"

rem_trg_mnt="${rem_workd}/trg_d"

rem_dl_d="${rem_workd}/dl"

rem_funcs="${rem_workd}/sh_funcs"

desc_instance="${TEMP_D}/instance.info"

vol_info="${TEMP_D}/volume.info"

vol_attach_info="${TEMP_D}/vol_attach.info"

vol_detach_info="${TEMP_D}/vol_detach.info"

vol_delete_info="${TEMP_D}/vol_delete.info"

create_snapshot_info="${TEMP_D}/create_snapshot.info"

desc_snapshot_poll="${TEMP_D}/desc_snapshot.poll"

register_info="${TEMP_D}/register.info"

vol_attach_dev=/dev/sdi

alt_attach_dev=/dev/xvdi

retry 3 5 xc2 info-instances "${REGARGS[@]}" "${iid}" > "${desc_instance}" ||

	fail "failed to get information about instance $iid"

if [ -z "$address" ]; then

	di_field host < "${desc_instance}" && remaddr=${_RET} &&

		[ -n "${remaddr}" ] || fail "failed to read hostname for ${iid}"

else

	remaddr="$address"

fi

di_field placement < "${desc_instance}" && placement=${_RET} &&

	[ -n "${placement}" ] || fail "unable to determine availability zone"

debug 2 "${iid} is at ${remaddr} in zone ${placement}"

debug 1 "verifying connection to ${remaddr}"

retry 15 30s xssh "${remaddr}" /bin/true ||

	fail "failed to verify connection to ${remaddr}"

# [host] attach a volume to the instance, wait for it to become available

retry 3 5 xc2 create-volume "${REGARGS[@]}" --size "$size" \

	--availability-zone "${placement}" > "${vol_info}" &&

	vol_id=$(awk '-F\t' '{print $2}' < "${vol_info}") && [ -n "${vol_id}" ] ||

	fail "failed to create a volume"

VOLUME_ID=${vol_id}

debug 2 "created volume ${vol_id}"

retry 15 30 xc2 attach-volume "${REGARGS[@]}" --device "${vol_attach_dev}" \

	--instance "${iid}" "${vol_id}" > "${vol_attach_info}" &&

	VOLUME_ATTACHED=1 ||

	(cat ${vol_attach_info} &&

	 get_volstate "$vol" &&

	 error "volume state is $_RET" &&

	 fail "failed to attach volume to ${vol_attach_dev}")

debug 2 "volume ${vol_id} attached to ${iid} at ${vol_attach_dev}"

# [instance] download the published tarfile from http://cloud-images.ubuntu.com

# [instance] extract image, mount loopback

getter="lftp"

debug 2 "obtaining image on ${remaddr}"

xssh "${remaddr}" bash -e <<EOF || fail "failed to download/setup image"

exec 1>&2

[ -e "${rem_workd}" ] && sudo rm -rf "${rem_workd}"

[ -e "${rem_dl_d}" ] && sudo rm -rf "${rem_dl_d}"

sudo mkdir -p "${rem_src_mnt}" "${rem_trg_mnt}" "${rem_dl_d}"

sudo chown -R \$(id -u) "${rem_workd}"

cat > "${rem_funcs}" <<"END"

error() { echo "$PREFIX\$@" 1>&2; }

fail() { [ \$# -eq 0 ] || error "\$@" 1>&2; exit 1; }

[ ${VERBOSITY:-0} -ge 2 ] || exec 2>/dev/null

[ ${VERBOSITY:-0} -ge 3 ] && set -x

exec 1>&2

END

. ${rem_funcs}

out="${rem_dl_d}/img.tar.gz"

if [ ! -f "\${out}" ]; then

	start=\${SECONDS}

	case "$getter" in

		lftp)

			dpkg-query --show lftp >/dev/null 2>&1 ||

				sudo env DEBIAN_FRONTEND=noninteractive \

					apt-get install lftp -qq >/dev/null

			imgurl="${imgurl}"

			proto=\${imgurl%%://*}

			site=\${imgurl#\${proto}://}; site=\${site%%/*};

			path=\${imgurl#\${proto}://\${site}/}

			lftp -c "open \${proto}://\${site}/ && pget \${path} -o \${out}" </dev/null

			;;

		*)

			wget "${imgurl}" --progress=dot:mega -O "\${out}"

			;;

	esac

	elapsed=\$((\${SECONDS}-\${start}))

	[ "\${elapsed}" = "0" ] && elapsed=1

	size=\$(stat --format "%s" "\${out}")

	rate=\$(((\${size}/1024)/\${elapsed}))

	error "got \${size} bytes in \${elapsed} seconds. \${rate} kB/s"

fi

if [ -n "${image_md5}" ]; then

	printf "%s  %s" "${image_md5}" "img.tar.gz" > "${rem_dl_d}/MD5SUMS"

	( cd "${rem_dl_d}" && md5sum --check MD5SUMS ) >/dev/null

fi

tar -C "${rem_dl_d}" -Sxzf "${rem_dl_d}/img.tar.gz"

sudo rm "${rem_dl_d}/img.tar.gz"

imgfile=""

for f in "${rem_dl_d}/"*; do

	case \$f in

		*.img) imgfile=\$f; break;;

	esac

done

[ -n "\$imgfile" ] || fail "failed to find *.img in tarball"

img_label=\$(e2label "\${imgfile}")

echo "img_label=\"\${img_label}\"" >> "${rem_funcs}"

sudo mount -o loop \${imgfile} "${rem_src_mnt}"

img_fstype=\$(awk '\$2 == "${rem_src_mnt}" { val=\$3 }; END { print val }' < /proc/mounts)

echo "img_fstype=\"\${img_fstype}\"" >> "${rem_funcs}"

EOF

# [instance] mkfs on attached volume

# [instance] copy data from loopbacked image to ebs volume, sync

if [[ "${etype}" =~ ssd ]]; then

	debug 2 "writing discard to rem_funcs ${rem_funcs}"

	write_discard | xssh "$remaddr" sh -c "cat >> ${rem_funcs}"

fi

if [[ "${etype}" =~ hvm ]]; then

	debug 2 "writing hvmify to rem_funcs ${rem_funcs}"

	write_hvmify | xssh "$remaddr" sh -c "cat >> ${rem_funcs}"

fi

debug 2 "copying data on ${remaddr}"

xssh "$remaddr" bash -e <<EOF || fail "failed to copy data to volume"

. ${rem_funcs}

i=0;

vol_attach_dev=${vol_attach_dev};

alt_attach_dev="${alt_attach_dev}";

while [ \$i -lt 180 ]; do

	[ -e "${vol_attach_dev}" ] && break ||

		{ [ -e "\${alt_attach_dev}" ] &&

			vol_attach_dev=\${alt_attach_dev} && break; } ||

		sleep 2

	i=\$((\$i+1))

done

[ -e "\${vol_attach_dev}" ] || {

	error "\${vol_attach_dev} doesn't exist, showing /proc/partitions"

	cat /proc/partitions 1>&2

	fail "\${vol_attach_dev} does not exist"

}

fsdev="\${vol_attach_dev}"

if [[ "${etype}" =~ hvm ]]; then

	echo "1,,L,*" | sudo sfdisk "\${vol_attach_dev}" ||

		fail "failed to partition \${vol_attach_dev}"

	fsdev="\${vol_attach_dev}1"

fi

label=\${img_label}

if [ -n "${fslabel}" ]; then

   [ "${fslabel}" = "none" ] && fslabel=""

   label="${fslabel}"

fi

xtype=\${img_fstype}

if [ -n "${fstype}" ]; then

   xtype="${fstype}"

fi

mkfs_args="-t \${xtype}"

case "\${xtype}" in

	ext*) mkfs_args="\${mkfs_args} -F";;

esac

error "creating fs: \$mkfs_args label=\${label} on \$fsdev"

sudo mkfs \${mkfs_args} \${label:+-L "\${label}"} "\${fsdev}" >/dev/null

sudo mount "\${fsdev}" "${rem_trg_mnt}"

sudo rsync -aXHAS "${rem_src_mnt}/" "${rem_trg_mnt}"

suite=\$(sudo chroot ${rem_trg_mnt} lsb_release --codename --short)

if [[ ! "\${suite}" =~ (lucid|10.04) ]]; then

	# Only enable discard for Lucid and later

	if [[ "${etype}" =~ ssd ]]; then

		discard "${rem_trg_mnt}"

	fi

	if [[ "${etype}" =~ hvm ]]; then

		hvmify "${rem_trg_mnt}" "\${vol_attach_dev}"

	fi

fi

sudo umount -f "${rem_trg_mnt}"

sudo umount -f "${rem_src_mnt}"

sync

sudo rm -rf "${rem_src_mnt}"

sudo rm -rf "${rem_workd}"

error "copied data to \${vol_attach_dev}"

EOF

# [host] detach volume, create-snapshot, delete-volume

detach_vol "${vol_id}" "${vol_detach_info}" ||

	(get_volstate "$vol" &&

	 error "volume state is $_RET" &&

 	 fail "failed to detach volume")

VOLUME_ATTACHED=0

debug 2 "creating snapshot of ${vol_id}"

snap_start=${SECONDS}

retry 10 60 xc2 create-snapshot "${REGARGS[@]}" \

	"--description=${snapshot_desc}" \

	"${vol_id}" > "${create_snapshot_info}" &&

	snapshot_id=$(awk '-F\t' 'END{print $2}' < "${create_snapshot_info}") &&

	[ -n "${snapshot_id}" ] ||

	fail "failed to create snapshot of ${vol_id}"

SNAPSHOT_ID=${snapshot_id}

retry 5 60 delete_vol "${vol_id}" "${vol_delete_info}" ||

	fail "failed to delete volume ${vol_id}"

VOLUME_ID=""

failed_last=0

debug 2 "waiting for ${snapshot_id} to become complete"

# [host] wait for snapshot to leave pending

for((i=0;i<90;i++)); do

	retry 3 10 xc2 info-snapshots "${REGARGS[@]}" "${snapshot_id}" \

		> "${desc_snapshot_poll}" && failed_last=0 ||

		failed_last=$((${failed_last}+1))

	[ "${failed_last}" -lt 30 ] ||

		fail "describe-snapshots failed ${failed_last} times in a row"

	if [ ${failed_last} -eq 0 ]; then

		state=$(awk '-F\t' '{print $4}' < "${desc_snapshot_poll}")

		case "${state}" in

			pending) :;;

			completed) break;;

			error) fail "snapshot ${snapshot_id} is in state 'errror'";;

			*) error "WARNING: unknown snapshot state \"${state}\"";;

		esac

		debug 2 "snapshot state is $state after $((${SECONDS}-${snap_start}))s"

	fi

	sleep 15

done

debug 1 "snapshot complete after $((${SECONDS}-${snap_start})) seconds"

# [host] register

args=( "${REGARGS[@]}" "--architecture=${ec2_arch}" "--name=${register_name}" )

[ -n "${description}" ] && args[${#args[@]}]="--description=${description}"

[ -n "${kernel}" ]      && args[${#args[@]}]="--kernel=${kernel}"

[ -n "${ramdisk}" ]     && args[${#args[@]}]="--ramdisk=${ramdisk}"

# Note (2010-08-31):

#  passing --block-device-mapping as shown here does get block devices

#  mapped for the 2 explicit cases.  Other block devices will not be

#  present by default, though.  ie, on an m1.large, which should have 2

#  ephemeral block devices, you will get only 1.

#

#  if you do not pass *any* --block-device-mapping, you will get no

#  ephemeral block devices.

#

#  if you pass a long list:

#    --block-device-mapping sdb=ephemeral0

#    --block-device-mapping sdc=ephemeral1

#    --block-device-mapping sdd=ephemeral2

#    --block-device-mapping sde=ephemeral3

#

#  then you will get that all appropriate block devices for the given size

#  when you run it (ie, m1.large will have sdb and sdc), but the

#  metadata will also have entries for sdd and sde.  That fact will *not*

#  prevent you from attaching a volume as '--device /dev/sde'

#

#  the end result of all of this is that almost all ubuntu EBS instances

#  will only have a single ephemeral device node unless the user

#  changes arguments at launch time.

case "${ec2_arch}:${etype}" in

	x86_64:ebs*)

		args[${#args[@]}]="--block-device-mapping"

 		args[${#args[@]}]="/dev/sdb=ephemeral0"

		;;

	i386:ebs*)

		args[${#args[@]}]="--block-device-mapping"

 		args[${#args[@]}]="/dev/sda2=ephemeral0"

		;;

	x86_64:hvm*)

		args[${#args[@]}]="--block-device-mapping"

 		args[${#args[@]}]="/dev/sdb=ephemeral0"

		args[${#args[@]}]="--block-device-mapping"

 		args[${#args[@]}]="/dev/sdc=ephemeral1"

		[ "${sriov:-0}" -eq 1 ] &&

			 args[${#args[@]}]="--sriov=simple"

		;;

esac

# Handle the specific mappings

debug 1 "ec2-images2ebs: etype is ${etype}"

case "${etype}" in

	*ssd)

		# SSDs use the gp2 disk type

		args[${#args[@]}]="--block-device-mapping"

		args[${#args[@]}]="/dev/sda1=${snapshot_id}::true:gp2"

		;;

	*io1)

		# Provisioned IOPs use io1

		args[${#args[@]}]="--block-device-mapping"

		args[${#args[@]}]="/dev/sda1=${snapshot_id}::true:io1:200"

		;;

	*)

		args[${#args[@]}]="--snapshot"

		args[${#args[@]}]="${snapshot_id}"

		;;

esac

[[ "${etype}" =~ hvm ]] && args[${#args[@]}]="--virtualization-type=hvm"

debug 1 "registering: register ${args[*]}"

retry 15 30 xc2 ximages register "${args[@]}" > "${register_info}" &&

	new_ami=$(awk '-F\t' 'END{print $2}' < "${register_info}") &&

	[ -n "${new_ami}" ] && SNAPSHOT_ID="" ||

	fail "failed to register: xc2 register ${args[*]}"

debug 1 "published ${register_name} as ${new_ami}"

echo "${new_ami}"

exit 0