~smoser/curtin/yakkety.lp1666986

#!/bin/bash

VERBOSITY=0
TEMP_D=""
HTTP_PID=""
XKVM_PID=""
HTTP_PORT_TRIES=${HTTP_PORT_TRIES:-5}
HTTP_PORT_MIN=${HTTP_PORT_MIN:-12000}
HTTP_PORT_MAX=${HTTP_PORT_MAX:-65500}
MY_D=$(dirname "$0")
DEFAULT_ROOT_ARG="root=LABEL=cloudimg-rootfs"

error() { echo "$@" 1>&2; }

Usage() {
    cat <<EOF
Usage: ${0##*/} [ options ] boot-image curtin install [args]

   boot the image 'boot-image', so that it will run
      curtin install [args]
   after booting. 'curtin install [args]' can be any command'

   options:
           --add  F[:T] add file 'F' to the curtin archive at T
      -a | --append     append args to kernel cmdline (--kernel)
      -A | --arch   A   assume guest kernel architecture A
      -d | --disk   D   add a disk 'D' format
                        (path[:size][:driver][:bsize][:devopts])
                        driver can be specified without size using path::driver
                        driver defaults to virtio-blk
                        bsize <logical>[,<physical>][,<min_io_size>]
                        bsize defaults to 512b sector size
                        opts is a comma delimitted list of property=value
                        elements. Examine qemu-kvm -device scsi-hd,? for
                        details.
           --vnc D      use -vnc D (mutually exclusive with --silent)
      -h | --help       show this message
      -i | --initrd F   use initramfs F
      -k | --kernel F   use kernel K
           --mem    K   memory in Kb
      -n | --netdev     netdev can be 'user' or a bridge
      -p | --publish F  make file 'F' available in web server
           --silent     use -nographic
           --vnc D      use -vnc D (mutually exclusive with --silent)
                        directly through to qemu-system.
                        Note, qemu adds 5900 to port numbers. (:0 = port 5900)
           --serial-log F  : log to F (default 'serial.log')
           --root-arg X pass 'X' through as the root= param when booting a
                        kernel.  default: $DEFAULT_ROOT_PARAM
      -v | --verbose    be more verbose
           --no-install-deps  do not install insert '--install-deps'
                              on curtin command invocations

    the following are passed through to xkvm:
        --uefi-nvram
        --bios

   use of --kernel/--initrd will seed cloud-init via cmdline
   rather than the local datasource

   Example:
    * boot myboot.img, and install my-root.tar.gz
      ${0##*/} myboot.img --publish my-root.tar.gz curtin \
         install PUBURL/my-root.tar.gz

EOF
}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
cleanup() {
    local msg="" pid=""
    [ ! -d "$TEMP_D" ] || msg="${msg:+$msg }remove temp dir ${TEMP_D}."
    [ -z "$HTTP_PID" ] || msg="${msg:+$msg }kill http pid ${HTTP_PID}."
    [ -z "$XKVM_PID" ] || msg="${msg:+$msg }kill xkvm pid ${XKVM_PID}."
    debug 1 "cleaning up [${SECONDS}s].${msg:+ $msg}"
    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
    for pid in ${XKVM_PID} ${HTTP_PID}; do
        kill $pid
    done
}
sighandle() {
    debug 1 "recieved $1"
    exit ${2:-1}
}

register_signal_handlers() {
    local cur
    for cur in TERM INT; do
        trap "sighandle $cur" "SIG$cur"
    done
}

debug() {
    local level=${1}; shift;
    [ "${level}" -gt "${VERBOSITY}" ] && return
    error "${@}"
}

get_my_ip() {
    [ -z "$IP_ADDR" ] || { _RET="${IP_ADDR}"; return 0; }
    local Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
    local iface ipaddr="" tmpf=""
    # work around LP: #1483440
    cp "/proc/net/route" "${TEMP_D}/current-route"
    while read Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT; do
        [ "$Mask" = "00000000" ] && break
    done < "${TEMP_D}/current-route"
    iface="$Iface"
    ipaddr=$(LC_ALL=C /sbin/ip -4 addr list dev "$iface" scope global) || return
    ipaddr=${ipaddr#* inet }
    ipaddr=${ipaddr%%/*}
    _RET="$ipaddr"
}

write_metadata() {
    cat <<EOF
instance-id: 'inst-${RANDOM}'
EOF
}

write_pstate_config() {
    local pstate="$1" config1="$2" config2="$3"
    cat > "$config1" <<EOF
#cloud-config
power_state:
  mode: "$pstate"
EOF
    sed -e "s,PSTATE,$pstate," > "$config2" <<"EOF"
#upstart-job
# precise does not do cloud-config poweroff
description "power-state for precise"
start on stopped cloud-final
console output
task
script
  [ "$(lsb_release -sc)" = "precise" ] || exit 0
  target="PSTATE"
  msg="precise-powerstate: $target"
  case "$target" in
     on) exit 0;;
     off|poweroff) shutdown -P now "$msg";;
     reboot) shutdown -r now "$msg";;
     *) echo "$msg : unknown target"; exit 1;;
  esac
  echo "$msg"
  exit 0
end script
EOF
}

write_userdata() {
    local x
    cat <<EOF
#cloud-config-archive
- type: text/cloud-config
  content: |
   password: passw0rd
   chpasswd: { expire: False }
   output: {all: '| tee -a /var/log/cloud-init-output.log'}
EOF
    for x in "$@"; do
        printf "%s\n" "- |" && sed 's,^,  ,' "$x" || return
    done
}

xkvm_check() {
    command -v xkvm >/dev/null 2>&1 && return
    [ -x "$MY_D/xkvm" ] && PATH="$MY_D:$PATH" && return
    cat 1>&2 <<EOF
Unable to find 'xkvm' in PATH, should be in curtin's tools/ dir
EOF
    return 1
}

_start_http() {
    local pubdir="$1" ip="$2" port="$3"
    local statfile="_datesum" contents="" burl="http://$ip:$port"
    local found="" hpid="" ret=""
    contents="$$:$(date +%s.%N)"
    echo "$contents" > "$pubdir/$statfile"
    ( set -eb; cd "$pubdir" &&
        exec python -m SimpleHTTPServer "$port" ) >"$pubdir/ws.log" 2>&1 &
    hpid=$!
    HTTP_PID=$hpid  # set so cleanup cleans up during wget
    debug 3 "checking web service [pid=$hpid] on $burl"
    found=$(env -u http_proxy wget -q --waitretry=0.4 --retry-connrefused \
        --tries=10 "$burl/$statfile" --timeout=4 -O - 2>/dev/null) &&
        [ "$found" = "$contents" ] && {
            _RET=$hpid
            return 0
        }
    ret=$?
    kill $hpid && HTTP_PID=""
    return $ret
}

start_http() {
    # start_http(pubdir, ip, port="", tries=5)
    # starts a web service at 'port' that serves files in 'pubdir'
    # waits until it is verified to be lisenting at ip
    # if port is not provided, '$tries' random ports are tried.
    #
    # sets HTTP_PID and returns in _RET the port selected.
    local pubdir="$1" ip="$2" port="$3" tries="${4:-5}" i=""
    [ -z "$ip" ] && ip="localhost"
    local pmin="$HTTP_PORT_MIN" pmax="$HTTP_PORT_MAX" ret="" tried=""
    local ptries=""
    ptries=( )
    if [ -n "$port" ]; then
        ptries=( $port )
    elif [ $(($pmax-$pmin+1)) -le $tries ]; then
        # if tries spans the whole range, then just try them all
        local range=$(($pmax-$pmin+1))
        for((i=0;i<$range;i++)); do
            ptries[$i]=$((pmin+$i))
        done
    else
        for((i=0;i<$tries;i++)); do
            ptries[$i]="random"
        done
    fi
    for port in "${ptries[@]}"; do
        [ "$port" = "random" ] && port=$(($pmin+($RANDOM%($pmax+1-$pmin))))
        debug 2 "trying http server $ip:$port"
        _start_http "$pubdir" "$ip" "$port" &&
            HTTP_PID="$_RET" && _RET="$port" &&
            debug 1 "serving $pubdir at http://$ip:$port/ in pid $HTTP_PID" &&
            return 0
        ret=$?
        tried="$tried $port"
    done
    error "failed to start http on service on $ip tried ports: ${tried# }"
    return $ret
}

find_apt_proxy() {
    # pick an apt proxy for the guest
    local out=""

    # if user set uncommon 'apt_proxy', then trust it
    [ -n "$apt_proxy" ] && echo "$apt_proxy" && return 0

    # see if the host has an apt proxy configured, and use it
    if command -v apt-config >/dev/null 2>&1; then
        out=$(apt-config shell x Acquire::HTTP::Proxy) &&
        out=$(sh -c 'eval $1 && echo $x' -- "$out") && [ -n "$out" ] &&
        echo "$out" && return
    fi

    return 1
}

main() {
    local short_opts="a:A:d:h:i:k:n:p:v"
    local long_opts="add:,append:,arch:,bios:,disk:,dowait,help,initrd:,kernel:,mem:,netdev:,no-dowait,power:,publish:,root-arg:,silent,serial-log:,uefi-nvram:,verbose,vnc:"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return 1; }

    local seed=""
    local bootimg="" bootimg_dist="" target="" mem="1024"
    local udata="" ip="" http_port="${HTTP_PORT}" burl=""
    local tmp="" top_d
    local initrd="" kernel="" uappend="" iargs="" disk_args=""
    local pubs="" disks="" pstate="null"
    local bsize="512"
    local netdevs="" install_deps="--install-deps"
    local arch_hint=""
    local video="-curses -vga std" serial_log="serial.log"
    local root_arg="$DEFAULT_ROOT_ARG"
    # dowait: run xkvm with a '&' and then 'wait' on the pid.
    #  the reason to do this or not do this has to do with interactivity
    #  if detached with &, then user input will not go to xkvm.
    #  if *not* detached, then signal handling is blocked until
    #  the foreground subprocess returns. which means we can't handle
    #  a sigterm and kill xkvm.
    #  We default to dowait=false if input and output are a terminal
    local dowait=""
    [ -t 0 -a -t 1 ] && dowait=false || dowait=true
    pubs=( )
    disks=( )
    addfiles=( )
    netdevs=( )
    pt=( )

    # if output is to a terminal, then set dowait default to false
    [ -t 0 ] && dowait=false || dowait=true
    while [ $# -ne 0 ]; do
        cur=${1}; next=${2};
        case "$cur" in
               --add) addfiles[${#addfiles[@]}]="$next"; shift;;
            -a|--append) uappend="$next"; shift;;
            -A|--arch) arch_hint="$next"; shift;;
            -d|--disk) disks[${#disks[@]}]="$next"; shift;;
               --dowait) pt[${#pt[@]}]="$cur"; dowait=true;;
            -h|--help) Usage ; exit 0;;
            -i|--initrd) initrd="$next"; shift;;
            -k|--kernel) kernel="$next"; shift;;
               --mem) mem="$next"; shift;;
            -n|--netdev) netdevs[${#netdevs[@]}]="$next"; shift;;
               --no-dowait) pt[${#pt[@]}]="$cur"; dowait=false;;
               --no-install-deps) install_deps="";;
               --power)
                case "$next" in
                    off) pstate="poweroff";;
                    on|none)  pstate="null";;
                    reboot) pstate="$next";;
                    *) error "Invalid power state, must be: off, on, reboot";;
                esac
                shift;;
            -p|--publish) pubs[${#pub[@]}]="$next"; shift;;
               --root-arg) root_arg="$next";;
               --serial-log) serial_log="$next"; shift;;
               --silent) video="-nographic";;
            --uefi-nvram|--bios)
                # handle all --opt=* pass through here.
                pt[${#pt[@]}]="$cur=$next";;
            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
               --vnc)
                    video="-vnc $next"
                    debug 1 "VNC requested - $next"
                    shift;;
            --) shift; break;;
        esac
        shift;
    done

    # handle passing through '-v' if given
    local t=""
    for((i=0;i<${VERBOSITY};i++)); do t="${t}v"; done
    [ -n "$t" ] && pt[${#pt[@]}]="-$t"

    [ $# -ge 0 ] || { bad_Usage "must provide boot-image"; return 1; }
    bootimg_dist="$1"
    shift
    cmdargs=( "$@" )

    xkvm_check || return
    TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
        { error "failed to make tempdir"; return 1; }

    trap cleanup EXIT
    register_signal_handlers

    if [ "${#disks[@]}" -eq 0 ]; then
        disks=( "${TEMP_D}/disk1.img" )
    fi

    bootimg_dist=$(readlink -f "$bootimg_dist") ||
        { error "bad bootimg $bootimg_dist"; return 1; }

    [ -z "$initrd" -o -f "$initrd" ] ||
        { error "initrd not a file: $initrd"; return 1; }
    [ -z "$kernel" -o -f "$kernel" ] ||
        { error "kernel not a file: $kernel"; return 1; }

    tmp=$(dirname "$0") && top_d=$(cd "$tmp" && cd .. && pwd) ||
        { error "failed to get dir for $0"; return 1; }

    local disk="" src="" size="" fmt="" out="" id="" driver="" if=""
    local split_input="" serial=""
    disk_args=( )
    id=1
    for disk in "${disks[@]}"; do
        ((id++))
        # 1=src
        # 2=src:size
        # 3=src:size:driver
        # 4=src:size:driver:bsize
        # 5=src:size:driver:bsize:devopts
        src=$(echo $disk | awk -F: '{print $1}')
        size=$(echo $disk | awk -F: '{print $2}')
        driver=$(echo $disk | awk -F: '{print $3}')
        bsize=$(echo $disk | awk -F: '{print $4}')
        devopts=$(echo $disk | awk -F: '{print $5}')

        if [ -z "${src}" ]; then
            error "Failed to provide disk source"
            exit 1
        fi

        if [ -z "${size}" ]; then
            size=5G
        fi

        if [ -z "${driver}" ]; then
            driver="virtio-blk"
        fi

        if [ -z "${bsize}" ]; then
            bsize="512"
        fi

        if [ ! -f "$src" ]; then
            qemu-img create -f raw "${src}" "$size" ||
                { error "failed create $src of size $size"; return 1; }
            fmt="raw"
        else
            out=$(LANG=C qemu-img info "$src") &&
                fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') ||
                { error "failed to determine format of $src"; return 1; }
        fi

        # prepend comma if passing devopts
        if [ -n "${devopts}" ]; then
            devopts=",${devopts}"
        fi

        # set logical/physical size blocksz is logical:phys
        local logbs=$(round_up ${bsize%%:*})
        local phybs=$(round_up ${bsize##*:})
        local bs_args="logical_block_size=$logbs"
              bs_args="${bs_args},physical_block_size=$phybs"
              bs_args="${bs_args},min_io_size=$logbs"

        disk_args=( "${disk_args[@]}" "-drive"
            "file=${src},if=none,cache=unsafe,format=$fmt,id=drv${id},index=$id" )

        disk_args=( "${disk_args[@]}" "-device"
            "${driver},drive=drv${id},${bs_args}${devopts}" )

    done

    get_my_ip || { error "failed to get your ip. set IP_ADDR"; return 1; }
    ip=${_RET}

    local tok src pub fpath
    # tok in pubs looks like file[:pubname]
    # link them into the temp dir for publishing
    for tok in "${pubs[@]}"; do
        case "$tok" in
            *:*) src="${tok%:*}"; pub="${tok##*:}";;
            *) src=${tok}; pub="";;
        esac
        fpath=$(readlink -f "$src") ||
            { error "'$src': failed to get path"; return 1; }
        if [ -n "$pub" ]; then
            pub="${src##*/}"
        fi
        ln -sf "$fpath" "${TEMP_D}/${pub}"
    done

    start_http "${TEMP_D}" "$ip" "$http_port" "${HTTP_TRIES}" </dev/null ||
        { error "failed to start http service"; return 1; }
    http_port=$_RET
    burl="http://$ip:${http_port}/"
    # now replace PUBURL anywhere in cmdargs
    for((i=0;i<${#cmdargs[@]};i++)); do
        cmdargs[$i]=${cmdargs[$i]//PUBURL/$burl}
    done

    local addargs="" f=""
    addargs=( )
    for f in "${addfiles[@]}"; do
        if [ "${f#*:}" != "$f" ]; then
            addargs[${#addargs[@]}]="--add=$f"
        else
            addargs[${#addargs[@]}]="--add=$f:${f##*/}"
        fi
    done

    # We now pack up --config= options for the user
    # potentially should check to make sure they've not already done this
    # as this updating could then be destructive / annoying
    # specifically, it could be annoying if you had a config inside
    # the image that you were meaning to reference, and we copied (or tried)
    # the file from the host.
    for((i=1;i<${#cmdargs[@]};i++)); do
        cur=${cmdargs[$i]}
        next=${cmdargs[$i+1]}
        stuffed_cfg=""
        fpath=""
        case "$cur" in
            --config|-c)
                fpath=$next;
                cmdargs[$i+1]="config/${fpath##*/}"
                ;;
            --config=*)
                fpath=${cur#--config=}
                cmdargs[$i]="--config=config/${fpath##*/}"
                ;;
            *) continue;;
        esac
        addargs[${#addargs[@]}]="--add=config/${fpath##*/}:$fpath"
    done

    if [ "${cmdargs[0]}" = "curtin" -a -n "$install_deps" ]; then
        # if this is a 'curtin' command, then also insert --install-deps
        debug 1 "adding --install-deps to command"
        cmdargs=( curtin $install_deps "${cmdargs[@]:1}" )
    fi

    PYTHONPATH="${top_d}${PYTHONPATH:+${PYTHONPATH}}" \
        "${top_d}/bin/curtin" pack "${addargs[@]}" -- \
        "${cmdargs[@]}" > "${TEMP_D}/install-cmd" ||
        { error "failed to pack"; return 1; }

    udata="${TEMP_D}/user-data"
    mdata="${TEMP_D}/meta-data"

    local ccfiles=""
    ccfiles=( )
    if [ -n "${pstate}" ]; then
        write_pstate_config "$pstate" "${TEMP_D}/pstate.1" "${TEMP_D}/pstate.2"
        ccfiles[${#ccfiles[@]}]="${TEMP_D}/pstate.1"
        ccfiles[${#ccfiles[@]}]="${TEMP_D}/pstate.2"
    fi

    if tmp=$(find_apt_proxy); then
        debug 1 "using $tmp for proxy"
        printf '#cloud-config\napt_proxy: "%s"\n' "$tmp" > "${TEMP_D}/cc-proxy"
        ccfiles[${#ccfiles[@]}]="${TEMP_D}/cc-proxy"
    fi

    if command -v ssh-keys-list >/dev/null 2>&1; then
        ssh-keys-list cloud-config > "${TEMP_D}/cc-ssh-keys" &&
            ccfiles[${#ccfiles[@]}]="${TEMP_D}/cc-ssh-keys" ||
            { error "failed to get users ssh keys."; return 1; }
    fi

    write_metadata > "$mdata" || { error "failed to write meta-data"; return 1; }
    write_userdata "${TEMP_D}/install-cmd" "${ccfiles[@]}" > "$udata"  ||
        { error "failed to write user-data"; return 1; }

    bootimg="${TEMP_D}/boot.img"
    qemu-img create -f qcow2 -b "${bootimg_dist}" "$bootimg" 2G ||
        { error "failed create from ${bootimg_dist}"; return 1; }

    local seedargs=""
    seedargs=()
    if [ -n "$kernel" ]; then
        local append="" root=""
        # Note: root_arg is by default done by LABEL.  This assumes
        # a.) our root device is not multipath
        # b.) no other disks attached will have this LABEL
        # c.) the LABEL is in fact correct.
        # all of these assumptions are true under vmtest.
        if [ -z "$root_arg" ]; then
            debug 1 "WARN: root_arg is empty with kernel."
        fi
        append="${root_arg:+${root_arg} }ds=nocloud-net;seedfrom=$burl"

        local console_name=""
        case "${arch_hint}" in
            s390x) console_name="";;
            ppc64*) console_name="hvc0";;
            *) console_name="ttyS0";;
        esac
        if [ -n "$console_name" ]; then
            append="${append} console=${console_name}"
        fi
        append="${append} $uappend"
        seedargs=( "${seedargs[@]}" -kernel "$kernel" )
        [ -n "$initrd" ] && seedargs=( "${seedargs[@]}" -initrd "$initrd" )
        seedargs=( "${seedargs[@]}" -append "$append" )
    else
        seed="${TEMP_D}/seed.img"
        cloud-localds "$seed" "$udata" "$mdata" ||
            { error "failed cloud-localds"; return 1; }
        seedargs=( "-drive" "file=${seed},if=virtio,media=cdrom" )
    fi

    local netargs
    netargs=( )
    for dev in "${netdevs[@]}"; do
        netargs=( "${netargs[@]}" "--netdev=${dev}" )
    done

    local cmd serial_args="" chardev_arg=""
    [ "${serial_log}" = "none" ] && serial_log=""
    if [ -n "${serial_log}" ]; then
        if [ "${arch_hint}" = "s390x" ]; then
            if [ "${serial_log}" = "stdio" ]; then
                chardev_arg="stdio"
            else
                chardev_arg="file,path=${serial_log}"
            fi
                serial_args="-nodefaults -chardev ${chardev_arg},id=charconsole0 -device sclpconsole,chardev=charconsole0,id=console0"
        else
            serial_args="-serial file:${serial_log}"
            #debug mode serial_args="-serial ${serial_log} -monitor stdio"
        fi
    fi
    # -monitor stdio
    cmd=(
        xkvm "${pt[@]}" "${netargs[@]}" --
        -m ${mem} ${serial_args} ${video}
        -drive "file=$bootimg,if=none,cache=unsafe,format=qcow2,id=boot,index=0"
        -device "virtio-blk,drive=boot"
        "${disk_args[@]}"
        "${seedargs[@]}"
        )

    debug 1 "running with dowait=$dowait: ${cmd[*]}"
    local sstart=$SECONDS
    if $dowait; then
        "${cmd[@]}" &
        XKVM_PID=$!
        debug 1 "xkvm pid: $XKVM_PID. launch pid: $$"
        wait "${XKVM_PID}"
        ret=$?
        XKVM_PID=""
    else
        "${cmd[@]}"
        ret=$?
    fi
    debug 1 "xkvm returned $ret took $(($SECONDS-$sstart))"

    return $ret
}

random_wwn() {
    # wwn must be a int64, less than (1 << 63) - 1
    # we achieve this by combining 4 (1 << 15) ints
    printf "0x%04x%04x%04x%04x" $RANDOM $RANDOM $RANDOM $RANDOM
}

round_up() {
    local size="${1}"
    local multiple="${2:-512}"
    local max_size=$((32 * 1024)) # 32k max

    size=$(( (($size + $multiple - 1) / $multiple) * $multiple))
    if [ $size -gt $max_size ]; then
        echo $max_size
        return
    elif [ $size -lt $multiple ]; then
        echo $multiple
        return
    fi
    echo $size
}


test_start_http() {
    # run this like:
    #  HTTP_PORT_MIN=59000 HTTP_PORT_MAX=63001 ./tools/launch \
    #     /tmp/smfoo localhost
    VERBOSITY=3
    trap cleanup EXIT
    register_signal_handlers
    echo "mypid: $$"
    start_http "$@" || { ret=$?; echo "returned $ret"; return $ret; }
    ret=$?
    port=$_RET
    echo "pid $HTTP_PID is serving on $port"
    sleep ${SLEEPTIME:-3} &
    XKVM_PID=$!
    wait $XKVM_PID
    ret=$?
    XKVM_PID=""
    return $ret
}

main "$@"

# vi: ts=4 expandtab