~smoser/curtin/yakkety.lp1666986

#!/bin/bash

VERBOSITY=0

TEMP_D=""

HTTP_PID=""

XKVM_PID=""

HTTP_PORT_TRIES=${HTTP_PORT_TRIES:-5}

HTTP_PORT_MIN=${HTTP_PORT_MIN:-12000}

HTTP_PORT_MAX=${HTTP_PORT_MAX:-65500}

MY_D=$(dirname "$0")

DEFAULT_ROOT_ARG="root=LABEL=cloudimg-rootfs"

error() { echo "$@" 1>&2; }

Usage() {

    cat <<EOF

Usage: ${0##*/} [ options ] boot-image curtin install [args]

   boot the image 'boot-image', so that it will run

      curtin install [args]

   after booting. 'curtin install [args]' can be any command'

   options:

           --add  F[:T] add file 'F' to the curtin archive at T

      -a | --append     append args to kernel cmdline (--kernel)

      -A | --arch   A   assume guest kernel architecture A

      -d | --disk   D   add a disk 'D' format

                        (path[:size][:driver][:bsize][:devopts])

                        driver can be specified without size using path::driver

                        driver defaults to virtio-blk

                        bsize <logical>[,<physical>][,<min_io_size>]

                        bsize defaults to 512b sector size

                        opts is a comma delimitted list of property=value

                        elements. Examine qemu-kvm -device scsi-hd,? for

                        details.

           --vnc D      use -vnc D (mutually exclusive with --silent)

      -h | --help       show this message

      -i | --initrd F   use initramfs F

      -k | --kernel F   use kernel K

           --mem    K   memory in Kb

      -n | --netdev     netdev can be 'user' or a bridge

      -p | --publish F  make file 'F' available in web server

           --silent     use -nographic

           --vnc D      use -vnc D (mutually exclusive with --silent)

                        directly through to qemu-system.

                        Note, qemu adds 5900 to port numbers. (:0 = port 5900)

           --serial-log F  : log to F (default 'serial.log')

           --root-arg X pass 'X' through as the root= param when booting a

                        kernel.  default: $DEFAULT_ROOT_PARAM

      -v | --verbose    be more verbose

           --no-install-deps  do not install insert '--install-deps'

                              on curtin command invocations

    the following are passed through to xkvm:

        --uefi-nvram

        --bios

   use of --kernel/--initrd will seed cloud-init via cmdline

   rather than the local datasource

   Example:

    * boot myboot.img, and install my-root.tar.gz

      ${0##*/} myboot.img --publish my-root.tar.gz curtin \

         install PUBURL/my-root.tar.gz

EOF

}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }

cleanup() {

    local msg="" pid=""

    [ ! -d "$TEMP_D" ] || msg="${msg:+$msg }remove temp dir ${TEMP_D}."

    [ -z "$HTTP_PID" ] || msg="${msg:+$msg }kill http pid ${HTTP_PID}."

    [ -z "$XKVM_PID" ] || msg="${msg:+$msg }kill xkvm pid ${XKVM_PID}."

    debug 1 "cleaning up [${SECONDS}s].${msg:+ $msg}"

    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"

    for pid in ${XKVM_PID} ${HTTP_PID}; do

        kill $pid

    done

}

sighandle() {

    debug 1 "recieved $1"

    exit ${2:-1}

}

register_signal_handlers() {

    local cur

    for cur in TERM INT; do

        trap "sighandle $cur" "SIG$cur"

    done

}

debug() {

    local level=${1}; shift;

    [ "${level}" -gt "${VERBOSITY}" ] && return

    error "${@}"

}

get_my_ip() {

    [ -z "$IP_ADDR" ] || { _RET="${IP_ADDR}"; return 0; }

    local Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT

    local iface ipaddr="" tmpf=""

    # work around LP: #1483440

    cp "/proc/net/route" "${TEMP_D}/current-route"

    while read Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT; do

        [ "$Mask" = "00000000" ] && break

    done < "${TEMP_D}/current-route"

    iface="$Iface"

    ipaddr=$(LC_ALL=C /sbin/ip -4 addr list dev "$iface" scope global) || return

    ipaddr=${ipaddr#* inet }

    ipaddr=${ipaddr%%/*}

    _RET="$ipaddr"

}

write_metadata() {

    cat <<EOF

instance-id: 'inst-${RANDOM}'

EOF

}

write_pstate_config() {

    local pstate="$1" config1="$2" config2="$3"

    cat > "$config1" <<EOF

#cloud-config

power_state:

  mode: "$pstate"

EOF

    sed -e "s,PSTATE,$pstate," > "$config2" <<"EOF"

#upstart-job

# precise does not do cloud-config poweroff

description "power-state for precise"

start on stopped cloud-final

console output

task

script

  [ "$(lsb_release -sc)" = "precise" ] || exit 0

  target="PSTATE"

  msg="precise-powerstate: $target"

  case "$target" in

     on) exit 0;;

     off|poweroff) shutdown -P now "$msg";;

     reboot) shutdown -r now "$msg";;

     *) echo "$msg : unknown target"; exit 1;;

  esac

  echo "$msg"

  exit 0

end script

EOF

}

write_userdata() {

    local x

    cat <<EOF

#cloud-config-archive

- type: text/cloud-config

  content: |

   password: passw0rd

   chpasswd: { expire: False }

   output: {all: '| tee -a /var/log/cloud-init-output.log'}

EOF

    for x in "$@"; do

        printf "%s\n" "- |" && sed 's,^,  ,' "$x" || return

    done

}

xkvm_check() {

    command -v xkvm >/dev/null 2>&1 && return

    [ -x "$MY_D/xkvm" ] && PATH="$MY_D:$PATH" && return

    cat 1>&2 <<EOF

Unable to find 'xkvm' in PATH, should be in curtin's tools/ dir

EOF

    return 1

}

_start_http() {

    local pubdir="$1" ip="$2" port="$3"

    local statfile="_datesum" contents="" burl="http://$ip:$port"

    local found="" hpid="" ret=""

    contents="$$:$(date +%s.%N)"

    echo "$contents" > "$pubdir/$statfile"

    ( set -eb; cd "$pubdir" &&

        exec python -m SimpleHTTPServer "$port" ) >"$pubdir/ws.log" 2>&1 &

    hpid=$!

    HTTP_PID=$hpid  # set so cleanup cleans up during wget

    debug 3 "checking web service [pid=$hpid] on $burl"

    found=$(env -u http_proxy wget -q --waitretry=0.4 --retry-connrefused \

        --tries=10 "$burl/$statfile" --timeout=4 -O - 2>/dev/null) &&

        [ "$found" = "$contents" ] && {

            _RET=$hpid

            return 0

        }

    ret=$?

    kill $hpid && HTTP_PID=""

    return $ret

}

start_http() {

    # start_http(pubdir, ip, port="", tries=5)

    # starts a web service at 'port' that serves files in 'pubdir'

    # waits until it is verified to be lisenting at ip

    # if port is not provided, '$tries' random ports are tried.

    #

    # sets HTTP_PID and returns in _RET the port selected.

    local pubdir="$1" ip="$2" port="$3" tries="${4:-5}" i=""

    [ -z "$ip" ] && ip="localhost"

    local pmin="$HTTP_PORT_MIN" pmax="$HTTP_PORT_MAX" ret="" tried=""

    local ptries=""

    ptries=( )

    if [ -n "$port" ]; then

        ptries=( $port )

    elif [ $(($pmax-$pmin+1)) -le $tries ]; then

        # if tries spans the whole range, then just try them all

        local range=$(($pmax-$pmin+1))

        for((i=0;i<$range;i++)); do

            ptries[$i]=$((pmin+$i))

        done

    else

        for((i=0;i<$tries;i++)); do

            ptries[$i]="random"

        done

    fi

    for port in "${ptries[@]}"; do

        [ "$port" = "random" ] && port=$(($pmin+($RANDOM%($pmax+1-$pmin))))

        debug 2 "trying http server $ip:$port"

        _start_http "$pubdir" "$ip" "$port" &&

            HTTP_PID="$_RET" && _RET="$port" &&

            debug 1 "serving $pubdir at http://$ip:$port/ in pid $HTTP_PID" &&

            return 0

        ret=$?

        tried="$tried $port"

    done

    error "failed to start http on service on $ip tried ports: ${tried# }"

    return $ret

}

find_apt_proxy() {

    # pick an apt proxy for the guest

    local out=""

    # if user set uncommon 'apt_proxy', then trust it

    [ -n "$apt_proxy" ] && echo "$apt_proxy" && return 0

    # see if the host has an apt proxy configured, and use it

    if command -v apt-config >/dev/null 2>&1; then

        out=$(apt-config shell x Acquire::HTTP::Proxy) &&

        out=$(sh -c 'eval $1 && echo $x' -- "$out") && [ -n "$out" ] &&

        echo "$out" && return

    fi

    return 1

}

main() {

    local short_opts="a:A:d:h:i:k:n:p:v"

    local long_opts="add:,append:,arch:,bios:,disk:,dowait,help,initrd:,kernel:,mem:,netdev:,no-dowait,power:,publish:,root-arg:,silent,serial-log:,uefi-nvram:,verbose,vnc:"

    local getopt_out=""

    getopt_out=$(getopt --name "${0##*/}" \

        --options "${short_opts}" --long "${long_opts}" -- "$@") &&

        eval set -- "${getopt_out}" ||

        { bad_Usage; return 1; }

    local seed=""

    local bootimg="" bootimg_dist="" target="" mem="1024"

    local udata="" ip="" http_port="${HTTP_PORT}" burl=""

    local tmp="" top_d

    local initrd="" kernel="" uappend="" iargs="" disk_args=""

    local pubs="" disks="" pstate="null"

    local bsize="512"

    local netdevs="" install_deps="--install-deps"

    local arch_hint=""

    local video="-curses -vga std" serial_log="serial.log"

    local root_arg="$DEFAULT_ROOT_ARG"

    # dowait: run xkvm with a '&' and then 'wait' on the pid.

    #  the reason to do this or not do this has to do with interactivity

    #  if detached with &, then user input will not go to xkvm.

    #  if *not* detached, then signal handling is blocked until

    #  the foreground subprocess returns. which means we can't handle

    #  a sigterm and kill xkvm.

    #  We default to dowait=false if input and output are a terminal

    local dowait=""

    [ -t 0 -a -t 1 ] && dowait=false || dowait=true

    pubs=( )

    disks=( )

    addfiles=( )

    netdevs=( )

    pt=( )

    # if output is to a terminal, then set dowait default to false

    [ -t 0 ] && dowait=false || dowait=true

    while [ $# -ne 0 ]; do

        cur=${1}; next=${2};

        case "$cur" in

               --add) addfiles[${#addfiles[@]}]="$next"; shift;;

            -a|--append) uappend="$next"; shift;;

            -A|--arch) arch_hint="$next"; shift;;

            -d|--disk) disks[${#disks[@]}]="$next"; shift;;

               --dowait) pt[${#pt[@]}]="$cur"; dowait=true;;

            -h|--help) Usage ; exit 0;;

            -i|--initrd) initrd="$next"; shift;;

            -k|--kernel) kernel="$next"; shift;;

               --mem) mem="$next"; shift;;

            -n|--netdev) netdevs[${#netdevs[@]}]="$next"; shift;;

               --no-dowait) pt[${#pt[@]}]="$cur"; dowait=false;;

               --no-install-deps) install_deps="";;

               --power)

                case "$next" in

                    off) pstate="poweroff";;

                    on|none)  pstate="null";;

                    reboot) pstate="$next";;

                    *) error "Invalid power state, must be: off, on, reboot";;

                esac

                shift;;

            -p|--publish) pubs[${#pub[@]}]="$next"; shift;;

               --root-arg) root_arg="$next";;

               --serial-log) serial_log="$next"; shift;;

               --silent) video="-nographic";;

            --uefi-nvram|--bios)

                # handle all --opt=* pass through here.

                pt[${#pt[@]}]="$cur=$next";;

            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;

               --vnc)

                    video="-vnc $next"

                    debug 1 "VNC requested - $next"

                    shift;;

            --) shift; break;;

        esac

        shift;

    done

    # handle passing through '-v' if given

    local t=""

    for((i=0;i<${VERBOSITY};i++)); do t="${t}v"; done

    [ -n "$t" ] && pt[${#pt[@]}]="-$t"

    [ $# -ge 0 ] || { bad_Usage "must provide boot-image"; return 1; }

    bootimg_dist="$1"

    shift

    cmdargs=( "$@" )

    xkvm_check || return

    TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||

        { error "failed to make tempdir"; return 1; }

    trap cleanup EXIT

    register_signal_handlers

    if [ "${#disks[@]}" -eq 0 ]; then

        disks=( "${TEMP_D}/disk1.img" )

    fi

    bootimg_dist=$(readlink -f "$bootimg_dist") ||

        { error "bad bootimg $bootimg_dist"; return 1; }

    [ -z "$initrd" -o -f "$initrd" ] ||

        { error "initrd not a file: $initrd"; return 1; }

    [ -z "$kernel" -o -f "$kernel" ] ||

        { error "kernel not a file: $kernel"; return 1; }

    tmp=$(dirname "$0") && top_d=$(cd "$tmp" && cd .. && pwd) ||

        { error "failed to get dir for $0"; return 1; }

    local disk="" src="" size="" fmt="" out="" id="" driver="" if=""

    local split_input="" serial=""

    disk_args=( )

    id=1

    for disk in "${disks[@]}"; do

        ((id++))

        # 1=src

        # 2=src:size

        # 3=src:size:driver

        # 4=src:size:driver:bsize

        # 5=src:size:driver:bsize:devopts

        src=$(echo $disk | awk -F: '{print $1}')

        size=$(echo $disk | awk -F: '{print $2}')

        driver=$(echo $disk | awk -F: '{print $3}')

        bsize=$(echo $disk | awk -F: '{print $4}')

        devopts=$(echo $disk | awk -F: '{print $5}')

        if [ -z "${src}" ]; then

            error "Failed to provide disk source"

            exit 1

        fi

        if [ -z "${size}" ]; then

            size=5G

        fi

        if [ -z "${driver}" ]; then

            driver="virtio-blk"

        fi

        if [ -z "${bsize}" ]; then

            bsize="512"

        fi

        if [ ! -f "$src" ]; then

            qemu-img create -f raw "${src}" "$size" ||

                { error "failed create $src of size $size"; return 1; }

            fmt="raw"

        else

            out=$(LANG=C qemu-img info "$src") &&

                fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') ||

                { error "failed to determine format of $src"; return 1; }

        fi

        # prepend comma if passing devopts

        if [ -n "${devopts}" ]; then

            devopts=",${devopts}"

        fi

        # set logical/physical size blocksz is logical:phys

        local logbs=$(round_up ${bsize%%:*})

        local phybs=$(round_up ${bsize##*:})

        local bs_args="logical_block_size=$logbs"

              bs_args="${bs_args},physical_block_size=$phybs"

              bs_args="${bs_args},min_io_size=$logbs"

        disk_args=( "${disk_args[@]}" "-drive"

            "file=${src},if=none,cache=unsafe,format=$fmt,id=drv${id},index=$id" )

        disk_args=( "${disk_args[@]}" "-device"

            "${driver},drive=drv${id},${bs_args}${devopts}" )

    done

    get_my_ip || { error "failed to get your ip. set IP_ADDR"; return 1; }

    ip=${_RET}

    local tok src pub fpath

    # tok in pubs looks like file[:pubname]

    # link them into the temp dir for publishing

    for tok in "${pubs[@]}"; do

        case "$tok" in

            *:*) src="${tok%:*}"; pub="${tok##*:}";;

            *) src=${tok}; pub="";;

        esac

        fpath=$(readlink -f "$src") ||

            { error "'$src': failed to get path"; return 1; }

        if [ -n "$pub" ]; then

            pub="${src##*/}"

        fi

        ln -sf "$fpath" "${TEMP_D}/${pub}"

    done

    start_http "${TEMP_D}" "$ip" "$http_port" "${HTTP_TRIES}" </dev/null ||

        { error "failed to start http service"; return 1; }

    http_port=$_RET

    burl="http://$ip:${http_port}/"

    # now replace PUBURL anywhere in cmdargs

    for((i=0;i<${#cmdargs[@]};i++)); do

        cmdargs[$i]=${cmdargs[$i]//PUBURL/$burl}

    done

    local addargs="" f=""

    addargs=( )

    for f in "${addfiles[@]}"; do

        if [ "${f#*:}" != "$f" ]; then

            addargs[${#addargs[@]}]="--add=$f"

        else

            addargs[${#addargs[@]}]="--add=$f:${f##*/}"

        fi

    done

    # We now pack up --config= options for the user

    # potentially should check to make sure they've not already done this

    # as this updating could then be destructive / annoying

    # specifically, it could be annoying if you had a config inside

    # the image that you were meaning to reference, and we copied (or tried)

    # the file from the host.

    for((i=1;i<${#cmdargs[@]};i++)); do

        cur=${cmdargs[$i]}

        next=${cmdargs[$i+1]}

        stuffed_cfg=""

        fpath=""

        case "$cur" in

            --config|-c)

                fpath=$next;

                cmdargs[$i+1]="config/${fpath##*/}"

                ;;

            --config=*)

                fpath=${cur#--config=}

                cmdargs[$i]="--config=config/${fpath##*/}"

                ;;

            *) continue;;

        esac

        addargs[${#addargs[@]}]="--add=config/${fpath##*/}:$fpath"

    done

    if [ "${cmdargs[0]}" = "curtin" -a -n "$install_deps" ]; then

        # if this is a 'curtin' command, then also insert --install-deps

        debug 1 "adding --install-deps to command"

        cmdargs=( curtin $install_deps "${cmdargs[@]:1}" )

    fi

    PYTHONPATH="${top_d}${PYTHONPATH:+${PYTHONPATH}}" \

        "${top_d}/bin/curtin" pack "${addargs[@]}" -- \

        "${cmdargs[@]}" > "${TEMP_D}/install-cmd" ||

        { error "failed to pack"; return 1; }

    udata="${TEMP_D}/user-data"

    mdata="${TEMP_D}/meta-data"

    local ccfiles=""

    ccfiles=( )

    if [ -n "${pstate}" ]; then

        write_pstate_config "$pstate" "${TEMP_D}/pstate.1" "${TEMP_D}/pstate.2"

        ccfiles[${#ccfiles[@]}]="${TEMP_D}/pstate.1"

        ccfiles[${#ccfiles[@]}]="${TEMP_D}/pstate.2"

    fi

    if tmp=$(find_apt_proxy); then

        debug 1 "using $tmp for proxy"

        printf '#cloud-config\napt_proxy: "%s"\n' "$tmp" > "${TEMP_D}/cc-proxy"

        ccfiles[${#ccfiles[@]}]="${TEMP_D}/cc-proxy"

    fi

    if command -v ssh-keys-list >/dev/null 2>&1; then

        ssh-keys-list cloud-config > "${TEMP_D}/cc-ssh-keys" &&

            ccfiles[${#ccfiles[@]}]="${TEMP_D}/cc-ssh-keys" ||

            { error "failed to get users ssh keys."; return 1; }

    fi

    write_metadata > "$mdata" || { error "failed to write meta-data"; return 1; }

    write_userdata "${TEMP_D}/install-cmd" "${ccfiles[@]}" > "$udata"  ||

        { error "failed to write user-data"; return 1; }

    bootimg="${TEMP_D}/boot.img"

    qemu-img create -f qcow2 -b "${bootimg_dist}" "$bootimg" 2G ||

        { error "failed create from ${bootimg_dist}"; return 1; }

    local seedargs=""

    seedargs=()

    if [ -n "$kernel" ]; then

        local append="" root=""

        # Note: root_arg is by default done by LABEL.  This assumes

        # a.) our root device is not multipath

        # b.) no other disks attached will have this LABEL

        # c.) the LABEL is in fact correct.

        # all of these assumptions are true under vmtest.

        if [ -z "$root_arg" ]; then

            debug 1 "WARN: root_arg is empty with kernel."

        fi

        append="${root_arg:+${root_arg} }ds=nocloud-net;seedfrom=$burl"

        local console_name=""

        case "${arch_hint}" in

            s390x) console_name="";;

            ppc64*) console_name="hvc0";;

            *) console_name="ttyS0";;

        esac

        if [ -n "$console_name" ]; then

            append="${append} console=${console_name}"

        fi

        append="${append} $uappend"

        seedargs=( "${seedargs[@]}" -kernel "$kernel" )

        [ -n "$initrd" ] && seedargs=( "${seedargs[@]}" -initrd "$initrd" )

        seedargs=( "${seedargs[@]}" -append "$append" )

    else

        seed="${TEMP_D}/seed.img"

        cloud-localds "$seed" "$udata" "$mdata" ||

            { error "failed cloud-localds"; return 1; }

        seedargs=( "-drive" "file=${seed},if=virtio,media=cdrom" )

    fi

    local netargs

    netargs=( )

    for dev in "${netdevs[@]}"; do

        netargs=( "${netargs[@]}" "--netdev=${dev}" )

    done

    local cmd serial_args="" chardev_arg=""

    [ "${serial_log}" = "none" ] && serial_log=""

    if [ -n "${serial_log}" ]; then

        if [ "${arch_hint}" = "s390x" ]; then

            if [ "${serial_log}" = "stdio" ]; then

                chardev_arg="stdio"

            else

                chardev_arg="file,path=${serial_log}"

            fi

                serial_args="-nodefaults -chardev ${chardev_arg},id=charconsole0 -device sclpconsole,chardev=charconsole0,id=console0"

        else

            serial_args="-serial file:${serial_log}"

            #debug mode serial_args="-serial ${serial_log} -monitor stdio"

        fi

    fi

    # -monitor stdio

    cmd=(

        xkvm "${pt[@]}" "${netargs[@]}" --

        -m ${mem} ${serial_args} ${video}

        -drive "file=$bootimg,if=none,cache=unsafe,format=qcow2,id=boot,index=0"

        -device "virtio-blk,drive=boot"

        "${disk_args[@]}"

        "${seedargs[@]}"

        )

    debug 1 "running with dowait=$dowait: ${cmd[*]}"

    local sstart=$SECONDS

    if $dowait; then

        "${cmd[@]}" &

        XKVM_PID=$!

        debug 1 "xkvm pid: $XKVM_PID. launch pid: $$"

        wait "${XKVM_PID}"

        ret=$?

        XKVM_PID=""

    else

        "${cmd[@]}"

        ret=$?

    fi

    debug 1 "xkvm returned $ret took $(($SECONDS-$sstart))"

    return $ret

}

random_wwn() {

    # wwn must be a int64, less than (1 << 63) - 1

    # we achieve this by combining 4 (1 << 15) ints

    printf "0x%04x%04x%04x%04x" $RANDOM $RANDOM $RANDOM $RANDOM

}

round_up() {

    local size="${1}"

    local multiple="${2:-512}"

    local max_size=$((32 * 1024)) # 32k max

    size=$(( (($size + $multiple - 1) / $multiple) * $multiple))

    if [ $size -gt $max_size ]; then

        echo $max_size

        return

    elif [ $size -lt $multiple ]; then

        echo $multiple

        return

    fi

    echo $size

}

test_start_http() {

    # run this like:

    #  HTTP_PORT_MIN=59000 HTTP_PORT_MAX=63001 ./tools/launch \

    #     /tmp/smfoo localhost

    VERBOSITY=3

    trap cleanup EXIT

    register_signal_handlers

    echo "mypid: $$"

    start_http "$@" || { ret=$?; echo "returned $ret"; return $ret; }

    ret=$?

    port=$_RET

    echo "pid $HTTP_PID is serving on $port"

    sleep ${SLEEPTIME:-3} &

    XKVM_PID=$!

    wait $XKVM_PID

    ret=$?

    XKVM_PID=""

    return $ret

}

main "$@"

# vi: ts=4 expandtab