~patviafore/vmbuilder/automated-ec2-builds

#!/bin/bash
# vi: ts=4 noexpandtab

## Copyright (C) 2011 Ben Howard <ben.howard@canonical.com> and Scott Moser <smoser@canonical.com>
## Date: 01 July 2011
##
## Script for doing builds with live-build system in an automated fashion
## This comes with ABSOLUTELY NO WARRANTY; for details see COPYING.
## This is free software, and you are welcome to redistribute it
## under certain conditions; see copyING for details.
base_d=$(dirname $(readlink -f "${0}"))
TEMP_D=""
FINAL_DIR=""
CHOWN_OPTS=""

error() { echo "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
debug() { error "$(date -R):" "$@"; }
cleanup() {
    if [ -n "${TEMP_D}" -a -d "${TEMP_D}" ]; then
        umount_r "${TEMP_D}" && rm -rf "${TEMP_D}" || return 1
    fi
    if [ -n "${FINAL_DIR}" -a -d "${FINAL_DIR}" -a -n "${CHOWN_OPTS}" ]; then
        chown -R "${CHOWN_OPTS}" "${FINAL_DIR}"
    fi
}

usage() {
cat << EOF
Live-Build Control Script
    Uses customized live build to produce Cloud Images

    Required Flags:
    --config    Config

    Optional
        --arch      Arch type, i.e. amd64 or i386, default all
        --dest      Directory to put the final image
        --image     What to name the image, defaults to root.img
        --rootlabel Over-ride configuration default name for rootfs label.
        --rootsize  Size of root fs in MM
        --owner     Set the owner of the resulting files
        --copy      Copy the final image for debugig to location
        --manifest  Where to drop the manfiest file
        --livepath  Location of live-build scripts
        --serial    image serial number, defaults to YYYYMMDD
        --hookscript Add a script to hooks script section
        --ec2-version Destription of EC2 image
        --propsoed  Build from propoposed
EOF
}

# Parse options

short_opts="d,l,h"
long_opts="arch:,dest:,part:,debug,owner:,copy:,manifest:,name:,suite:,version:,type:,rootlabel:,moniker:,image:,rootsize:,livepath:,serial:,hookscript:,ec2-version:,config:,proposed"
getopt_out=$(getopt --name "${0##*/}" \
    --options "${short_opts}" --long "${long_opts}" -- "$@") &&
    eval set -- "${getopt_out}" ||
    bad_Usage

arch_types="amd64 i386"
finaldir="/tmp/live-build-$(date +%Y%m%d)_$(uuidgen)" # This is usually overriden at runtime
host_lsb_release=$(lsb_release -c -s)
partfile=""
debug="0"
partfile=""
owner=""
copy_final=""
manifest=""
version=""
plist=""
build_type="server"
imagename="root.img"
description=""
distribution_override=""
root_label=""
root_size=""
hookscript=""
config=""
proposed=0
serial=$(date +%Y%m%d)
live_path="${LIVE_BUILD_PATH}"

while [ $# -ne 0 ]; do
   case $1 in
           --config)       config="$2";         shift;;
           --arch)         arch_types="$2";     shift;;
           --dest)         finaldir="$2";       shift;;
           --part)         partfile="$2";       shift;;
        -d|--debug)        debug=1;             shift;;
           --owner)        owner="$2";          shift;;
           --manifest)     manifest="$2";       shift;;
           --copy)         copy_final="$2";     shift;;
           --name|--suite) suite="$2";          shift;;
           --version)      version="$2";        shift;;
           --type)         build_type="$2";     shift;;
           --rootlabel)    root_label="$2";     shift;;
           --moniker)      moniker="$2";        shift;;
           --image)        imagename="$2";      shift;;
           --rootsize)     root_size="$2";      shift;;
           --livepath)     live_path="$2";      shift;;
           --serial)       serial="$2";         shift;;
           --hookscript)   hookscript="$2";     shift;;
           --ec2-version)  description="$2";    shift;;
           --proposed)     proposed=1;          shift;;

        -h|--help) Usage; exit 0;;
        --) shift; break;;
    esac
    shift;
done

#Done setting options

if [ "${debug}" != "0" ]; then
   set -x
fi

### INITIALIZATION
lb_config_src_loc="$(dirname $(readlink -f "${0}"))/live-config"

[ -n "${live_path}" ] || fail "must specify --livepath or set LIVE_BUILD_PATH environment"
[ -d "${live_path}" ] || fail "live_path: not a directory [$live_path]"


[ -n "${config}" ] || fail "must provide configuratoin file"
[ -e "${config}" ] || fail "configuration file does not exist"

. ${config}

# umount_r(mp) : unmount any filesystems under r
#  this is useful to unmount a chroot that had sys, proc ... mounted
umount_r() {
    local p
    for p in "$@"; do
        [ -n "$p" ] || continue
        tac /proc/mounts | sudo sh -c '
            p=$1
            while read s mp t opt a b ; do
                [ "${mp}" = "${p}" -o "${mp#${p}/}" != "${mp}" ] ||
                    continue
                umount "$mp" || exit 1
            done
            exit 0' umount_r "${p%/}"
        [ $? -eq 0 ] || return
    done
}

pids_using() {
    local ppids="" pids=""
    for path in "$@"; do
        pids=$(sudo lsof |
            awk '( $9 == mp || $9 ~ mps ) { printf("%s ",$2); }' \
                "mp=${path%/}" "mps=^${path%/}/" ) ||
            return 1
        pids=${pids% }
        ppids="${ppids}${pids:+ ${pids}}"
        ppids=${ppids# }
    done
    _RET=${ppids}
}

force_unuse_d() {
    local d="$1" i=0 tries=10 pids=""
    while pids_using "$d" && pids=${_RET} && [ -n "${pids}" ] &&
        [ $i -lt $tries ] && i=$(($i+1)); do
        error "killing pids occupying ${d}:"
        sudo ps -p ${pids} 1>&2
        sudo kill -9 ${pids}
        sleep 2
        pids=""
    done
    # if there are pids around, then we failed
    [ -z "${pids}" ]
}

get_owner_chown_opts() {
    local owner="$1" opts="${owner}:"
    shift;
    if [ -z "${owner}" -a -n "${SUDO_UID}" ]; then
        opts="${SUDO_UID}:${SUDO_GID}"
    elif [ -z "${owner}" -a -n "${SUDO_USER}" ]; then
        opts="${SUDO_USER}:$(id ${SUDO_USER} -g)"
    elif [ -n "${owner}" ]; then
        opts="$(id -u ${owner}):$(id -g ${owner})"
    else
        opts="$(id -u):$(id -g)"
    fi
    _RET=${opts}
    return
}

builder() {
    #Run live-build. Hopefully this works
    local base_d=${1} arch="${2}" imagename="${3}" out_d="${4}"
    local chown_opts=${5} copy_final=${6}
    local ret=0 start_d="$PWD"

    debug "Running builder"
    debug "  builder1) Image will be named ${imagename}"
    debug "  builder1) Building for ${arch}"
    debug "  builder1) build location is "${base_d}""

    local lbb="${base_d}/live-build" lb_env="" lbcmd=""
    lb_env=( "LB_BASE=${lbb}" "PATH=${lbb}/scripts/build:${PATH}"
        "${lbb}/scripts/build/lb" build )
    lb_cmd=( sudo "${lb_env[@]}" "${lbb}/scripts/build/lb" build )

    debug "  builder2) cmd: " "${lb_cmd[@]}"
    debug "  builder3) Starting build prcoess...this might take a while"

    debug "  _______ LIVE-BUILD OUTPUT _______"
    cd "${base_d}"
    "${lb_cmd[@]}";  ret=$?
    cd "${start_d}"
    debug "  _______ END LIVE-BUILD OUTPUT _____"

    if ! umount_r "${base_d}/chroot" "${base_d}/cloud"; then
        error "failed to unmount directories under ${base_d}";
        force_unuse_d "${base_d}/chroot" "${base_d}/cloud" &&
            umount_r "${base_d}/chroot" "${base_d}/cloud" &&
            error "  recovered via force_unuse_d"
        return 1;
    fi

    # if live-build failed, return failure
    [ ${ret} -eq 0 ] || { error "live-build failed"; return $ret; }

    # Move the image into place
    [ -e "${base_d}/binary-raw.img" ] ||
        { error "binary-raw.img did not exist in output directory"; return 1; }
    mv "${base_d}/binary-raw.img" "${out_d}/${imagename}"

    if [ ! -z "${copy_final}" ]; then
        cp "${base_d}/binary-raw.img" "${copy}"
        debug "  builder3) Copied final image to ${copy_final}"
    fi

    debug "  builder3) Binary cloud image can be found at ${out_d}/${imagename}"

    # Pull out the binary.packages
    if [ -e "${base_d}/binary.packages" ]; then
        cp "${base_d}/binary.packages" "${manifest}" || fail "Unable to copy ${base_d}/binary.packages to ${manifest}"
        sed -i "s|\t| |g" "${manifest}" || fail "Unable to normalize format of manifest file"
    else
        error "  builder3) !! ERROR !! Unable to find manifest"
    fi

    # Clean up after ourselves
    if [ -n "${chown_opts}" ]; then
        sudo chown -R "${chown_opts}" "${base_d}" "${out_d}" ||
            { error "failed to change ownership to ${owner}" ; return 1; }
    fi

}

# get the owner chmod opts set up
get_owner_chown_opts "$owner"
chown_opts=${_RET}
CHOWN_OPTS=${chown_opts}

trap cleanup EXIT
if [ ! -d "${finaldir}" ]; then
    mkdir -p "${finaldir}" 2> /dev/null || fail "UNABLE TO CREATE FINAL DIR ${finaldir}"
fi
TEMP_D=$(mktemp -d "${finaldir}/build-lb-tmpd.XXXXXX")

FINAL_DIR="${finaldir}"

for arch in ${arch_types}
do
    work_d="${TEMP_D}/$arch"
    conf_d="${work_d}/config"
    mkdir -p "${work_d}"


    # Dynamically configure LB
    {
        debug "Dynamically configuring live-build"
        cd ${work_d}

        lbb="${work_d}/live-build"
        lbb_path="${lbb}/scripts/build"
        mkdir -p "${lbb_path}"

        rsync -a "${live_path}/" "${work_d}/live-build" ||
            fail "failed to copy ${live_path}"

        new_path="PATH=${live_path}/scripts/build:${PATH}"

        # %d-> distro, %l-> hard disk label, %s-> size, %q-> qemu-static location
        lbcmd=( "${lb_conf[@]}" )
        lbcmd=( "${lbcmd[@]//%d/${suite}}")
        lbcmd=( "${lbcmd[@]//%l/${root_fs_label}}")
        lbcmd=( "${lbcmd[@]//%s/${root_fs_size}}")
        lbcmd=( "${lbcmd[@]//%P/${lbb_path}}")
        lbcmd=( "${lbcmd[@]//%L/${lbb}}")
        lbcmd=( "${lbcmd[@]//%x/${new_path}}")
        lbcmd=( "${lbcmd[@]//%A/${arch}}")

		# Build the chroot from -proposed
		[ "${proposed}" -eq 1 ] && lbcmd+=("--proposed=true")

        # Discover qemu location and set it for arm images
        [[ "${arch}" =~ (arm|arm64) ]] && {
                case ${arch} in
                        armel|armhf) q_arch="arm";;
                        arm64) q_arch="aarch64";;
                        *) q_arch="$arch";;
                esac
                debug "looking for qemu-${q_arch}-static"

                [ -e "${HOME}/ec2-daily/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ] &&
                qemu_exec="${HOME}/ec2-daily/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ||
                [ -e "${lb_config_src_loc}/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ] &&
                        qemu_exec="${lb_config_src_loc}/qemu-static/${host_lsb_release}/qemu-${q_arch}-static" ||
                        qemu_exec="$(which qemu-${q_arch}-static)" ||
                        fail "Unable to locate qemu-${q_arch}-static"

                lbcmd=( "${lbcmd[@]//%q/${qemu_exec}}" )
                debug "found ${qemu_exec}"
                debug "debootstrap will be foreign"
        }


        debug "lb_config commmand is: ${lbcmd[@]}"
        sudo "${lbcmd[@]}" ||
            fail "Failed to dynammicaly configure live-build"

    }

    # Update ubuntu-cloud definition for precise
    [ "${suite}" \< "precise" ] || {
        echo "update-notifier-common" >> "${work_d}/live-build/package-lists/ubuntu-cloud" ||
            fail "Unable append update-notifier-common to package definition"
    }

    # make config tempdir
    rsync -a "${lb_config_src_loc}/common/"  "${conf_d}" ||
        fail "failed to copy common hooks"

    # seed directories
    mkdir -p "${conf_d}/includes.chroot/etc/cloud"

    # write build.info
    echo "build_name: ${build_type}" >> "${conf_d}/includes.chroot/etc/cloud/build.info" &&
        echo "serial: ${serial}"  >> "${conf_d}/includes.chroot/etc/cloud/build.info" ||
            fail "Unable to write etc/cloud/build.info"

    # write ec2-version
    echo "${description}" >> "${conf_d}/includes.chroot/etc/ec2_version" ||
        fail "Unable to write ${conf_d}/includes.chroot/etc/ec2_version"

    [ -z "${hookscript}" ] || {
        local_hook="${conf_d}/hooks/999hookscript.chroot"

        cp -a "${hookscript}" "${local_hook}" ||
            fail "Unable to copy ${hookscript} to ${local_hook}"

        debug "Populated hookscript to: ${conf_d}/hooks"
    }

    builder "${work_d}" "${arch}" "${imagename}" "${finaldir}" \
        "${chown_opts}" "${copy_final}" ||
        fail "failed to build for ${suite}"

    rm -Rf "${work_d}" || fail "Unable to clean ${temp_d}"
done

# Cleanup
cleanup && TEMP_D="" && FINAL_DIR="" && CHOWN_OPTS="" ||
    fail "failed to cleanup"

debug "Images and logs can be found at "${finaldir}""