~fginther/vmbuilder/jenkins_kvm-add-git-for-maas

#!/bin/bash
#
# Copyright 2014, Ben Howard <ben.howard@canonical.com>
# Copyright 2014, Canonical Group, Ltd
#
# This is a runner script emulating the Ubuntu Cloud Image
# derivative builds.
nbd_dev=""
mnt_dev=""

clean_nbd() {
    [ -n "${mnt_dev}" ] && {
        sudo umount -f "${mnt_dev}" && unset mnt_dev &&
        debug "Unmounted ${mnt_dev}" ||
        error "BAD! ${mnt_dev} may still be mounted!";
    }
    sleep 3
    [ -n "${nbd_dev}" ] && {
        sudo qemu-nbd -d "${nbd_dev}" &&
        unset nbd_dev &&
        debug "Removed device ${nbd_dev}" ||
        error "BAD! ${nbd_dev} may still exist!";
    }
}
clean() { [ -d ${tmp_dir} ] && rm -rf ${tmp_dir};
          clean_nbd; exit "${@}"; }
error() { echo "$@"; }
fail() { debug "${1:-Something bad happend}"; clean 1; }
debug() { error "$(date -R):" "$@"; }

my_name=$(readlink -f ${0})
my_dir=$(dirname ${my_name})

usage() {
cat << EOF
This program is used to create derivative Ubuntu Cloud Images outside the
Canonical build environement.

This program must run on Ubuntu 12.04 or 14.04.

    --suite:     the Ubuntu Code name to build against
    --use_img:   use a local file instead of fetching a new image
    --install:   install dependencies of this builder
    --streams:   Simple streams URL to use (optional)
    --template:  template to use, defaults to default.tmpl
    --addin:     the name of the addin to the template file to
                 use.
    --arch:      arch to build, i.e. i386 or amd64, defaults to system
    --reuse:     Reuse checkouts, etc. from <DIR>
    --nocheckout: Don't checkout the latest bzr branch
    --resize:    Resize root disk to xGB
    --config:    Read values from configuration file
    --final_img: Name of final image
    --cloud_cfg: File of Cloud-init #cloud-config file
    --fetch_new: Fetch new image if missing
    --resize_final: Resize final image to xGB
    --proposed:  Install from proposed
    --ppa:       Add PPA for doing build
EOF
exit 1
}

host_suite=$(lsb_release --codename --short)
install_deps() {
    local pkgs=(bzr qemu-kvm simplestreams)
    [ "${host_suite}" == "precise" ] && {
        sudo add-apt-repository ppa:ubuntu-cloud-archive/cloud-tools-next ||
            fail "Failed to add require repo!"
    }
    debug "Installing dependencies"
    sudo apt-get update
    sudo apt-get -y install ${pkg[@]}
}

short_opts="h"
long_opts="suite:,use_img:,install,streams:,template:,addin:,arch:,resize:,config:,final_img:,cloud_cfg:,fetch_new,resize_final:,cloud-init-file:,proposed,ppa:"
getopt_out=$(getopt --name "${0##*/}" \
    --options "${short_opts}" --long "${long_opts}" -- "$@") &&
    eval set -- "${getopt_out}" ||
    usage

suite="${suite:-trusty}"
build_arch="${build_arch:-$(dpkg --print-architecture)}"
use_img="${use_img}"
main_template="${main_template:-templates/default.tmpl}"
addin_template=""
install="${install:-0}"
_streams="${streams:-https://cloud-images.ubuntu.com/releases/streams/v1/com.ubuntu.cloud:released:download.json}"
resize="${resize:-0}"
resize_final=""
config=""
cloud_cfg=""
fetch_new=0
proposed=0
ppa=""

while [ $# -ne 0 ]; do
    cur=${1}; next=${2};

    case "${cur}" in
        --suite)        suite="${2}"; shift;;
        --use_img)      use_img="${2}"; shift;;
        --install)      install_deps; shift;;
        --streams)      streams="${2}"; shift;;
        --template)     main_template="${2}"; shift;;
        --addin)        addin_template="${2}"; shift;;
        --arch)         build_arch="${2}"; shift;;
        --resize)       resize="${2}"; shift;;
        --config)       config="${2}"; shift;;
        --cloud_cfg)    cloud_cfg="${2}"; shift;;
        --final_img)    final_img="${2}"; shift;;
        --proposed)     proposed=1;;
        --ppa)          ppa=${2}; shift;;
        --fetch_new)    fetch_new=1;;
        --resize_final) resize_final="${2}"; shift;;
        --cloud-init-file) [ -z "${cloud_init_files}" ] &&
                             cloud_init_files="--cloud-init-file ${2}" ||
                             cloud_init_files="${cloud_init_files} --cloud-init-file  ${2}";
                             shift;;
        -h|--help)      usage; exit 0;;
        --) shift; break;;
    esac
    shift;
done

# Fly with the safety on!
trap fail EXIT
trap fail SIGINT

# Readin the config if its there
[ -n "${config}" -a -e "${config}" ] &&
    . "${config}" &&
    debug "Read in configuration from ${config}"

[ -e "${cloud_cfg}" ] ||
    fail "Unable to find cloud_cfg '${cloud_cfg}'"

[ -e "${main_template}" ] ||
    fail "Unable to find ${main_template}. You may need pass --template"
main_template=$(readlink -f ${main_template})
[ -n "${addin_template}" ] && addin_template=$(readlink -f ${addin_template})

# Create a temporary directory for the fun
tmp_dir=$(mktemp -d builder.XXXXX --tmpdir=${TMPDIR:-/tmp});

# Switch to the  daily streams for the development release
devel_suite=$(ubuntu-distro-info --devel)
if [ "${suite}" == "${devel_suite}" ]; then
    _streams="${_streams//released/daily}"
    _streams="${_streams//releases/daily}"
fi
streams="${streams:-$_streams}"

# Fetch the file
[ -z "${use_img}" -a -z "${suite}" ] &&
    fail "Must define --use_img or --suite"

[ ! -e "${use_img}" -a "${fetch_new:-0}" -eq 0 ] &&
    fail "Image ${use_img} does not exist!"

if [ ! -e "${use_img}" ]; then
    debug "Getting location of the lastest image"
    debug "   Looking for ${suite} / ${build_arch}"
    stream_out=$(sstream-query ${streams} \
                    release=${suite} arch=${build_arch} \
                    item_name=disk1.img \
                    --max=1 --output-format="%(item_url)s,%(sha256)s")
    img_url=${stream_out%%,*}
    sha256=${stream_out##*,}
    use_img="${use_img:-$tmp_dir/$suite-$(date +%Y%m%d-%s).img}"
    debug "Fetching image from ${img_url}"
    debug "   Local file is ${use_img}"
    wget -O "${use_img}" "${img_url}" &&
         debug "Fetched imaged" ||
         fail "Failed to fetch the image!"
elif [ -e "${use_img}" ]; then
    debug "Using image ${use_img}"
fi

# Set the default cloud_cfg if not set
cloud_cfg=${cloud_cfg:-$(readlink -f config/cloud-$suite.cfg)}

debug "Modifying upstream Cloud Image"

# Put the final bits in place
template_f="${tmp_dir}/new_template.txt"
raw_f="${tmp_dir}/raw_f-$(date +%s).img"
final_img="${final_img:-$PWD/final-$(date +%s).img}"
attach_img="${tmp_dir}/$(uuidgen)-attach.img"

# Just some information about the final image
cp "${use_img}" "${attach_img}" ||
    fail "failed to copy image to ${attach_img}"

[ "${resize_final:-0}" -ne 0 ] && {
    qemu-img resize "${attach_img}" "${resize_final}G" &&
        debug "resized final disk to ${resize_final}G" ||
        fail  "failed to resize final disk";
}

debug "Using ${attach_img} for working image"
debug "Using ${final_img} as the final disk"

# Create the template.
cur_dir=${PWD}
if [ -n "${addin_template}" ]; then
  cd ${tmp_dir}

  awk '/ADDIN_HERE/{n++}{print >"template" n ".txt" }' \
        ${main_template} ||
            fail "failed to split template!"

  cat template.txt ${addin_template} template1.txt > ${template_f};

  sed -e "s,ADDIN_HERE,# END Addins,g" \
      -e "s,%%PPA%%,${ppa},g" \
      -e "s,%%PROPOSED%%,${proposed:-0},g" \
      -i  ${template_f} ||
        fail "Unable to finalize template!"

  cd ${cur_dir}
else
    template_f="${main_template}"
fi

# We need to be safe with the image, so we will modify it
debug "Modifying the builder image to make it work"
debug "This requires root access"

# We need nbd0
(lsmod | grep nbd >> /dev/null) || {
    debug "Adding kernel module nbd (cmd: sudo modprobe nbd)";
    sudo modprobe nbd ||
        fail "Failed to add nbd kernel module";
}

# Create a mod disk for working with
mod_img="${tmp_dir}/mod_img.qcow2"
cp "${use_img}" "${mod_img}" || fail "unable to copy ${use_img} to ${mod_img}"
debug "Mounting builder image (cmd: sudo qemu-nbd -C /dev/nbd10 ${mod_img}"
sudo qemu-nbd -c /dev/nbd10 ${mod_img} && {
        nbd_dev="/dev/nbd10" && debug "Added nbd block /dev/nbd10"; } ||
        fail "failed to setup nbd device!"

# Now mount it
[ -b "${nbd_dev}p1" ] && mnt_dev="${nbd_dev}p1" || mnt_dev="${nbd_dev}"

# Give the filesystem a new label, because cloudimg-rootfs won't work
# when we have two file systems with that
new_label="bld-$(openssl rand -hex 3)"
debug "Changing root label to ${new_label}"
sudo tune2fs -L ${new_label} ${mnt_dev}

nbd_mnt="${tmp_dir}/mnt"
[ -d "${nbd_mnt}" ] || {
    mkdir -p  "${nbd_mnt}" ||
    fail "mount point for nbd device does not exist";
}
debug "Mounting ${mnt_dev} to ${nbd_mnt} (cmd: mount ${mnt_dev} ${nbd_mnt})"
sudo mount ${mnt_dev} ${nbd_mnt} &&
    debug "Mounted ${mnt_dev} to ${mnt_dev}" ||
    fail  "Failed to mount ${mnt_dev}"

# Now change some bits...
debug "Changing label of builder root fs"
sudo sed -i "s,cloudimg-rootfs,${new_label},g" \
            "${nbd_mnt}/etc/fstab" \
            "${nbd_mnt}/boot/grub/grub.cfg" \
            "${nbd_mnt}/boot/grub/menu.lst" &&
            debug "Modified image successfully" ||
            fail  "Failed to modify image"

# And unmount the stuff
clean_nbd
sleep 3

# This is the command that will actually do the build
mod_cmd=(
    "${my_dir}/launch_kvm.sh"
    "--user-data ${template_f}"
    "--cloud-config ${cloud_cfg}"
    "--disk-gb 5" "--mem 1024"
    "--extra-disk $(readlink -f $attach_img)"
    "--raw-disk $(readlink -f $raw_f)"
    "--raw-size ${raw_size:-20}"
    "--img-url $(readlink -f $mod_img)"
    "${cloud_init_files[@]}"
)

# Run the command
debug "Command will be ${mod_cmd[@]}"
${mod_cmd[@]} &&
    debug "Finished with building image" ||
    fail  "KVM instance indicates failure!"

debug "Compressing ${final_img}"
qemu-img convert -O qcow2 -c ${attach_img} ${final_img} ||
    fail "failed to compress ${final_img}"

# Clear the trap
trap - EXIT SIGINT
trap
clean 0