~ubuntu-on-ec2/ubuntu-on-ec2/ec2-publishing-scripts.hvm : revision 513

1

#!/bin/bash

2

# vi: ts=4 noexpandtab

3

4

TMPD=""

5

RENAME_D=""

6

EXTRACT_D=""

7

VERBOSITY=0

8

REGIONS=( )

9

[ -n "${#EC2_ALL_REGIONS}" ] && REGIONS=( ${EC2_ALL_REGIONS} )

10

11

DEFAULT_MAPPER=append_full

12

13

error() { echo "$@" 1>&2; }

14

errorp() { printf "$@" 1>&2; }

15

fail() { [ $# -eq 0 ] || error "$@"; exit 1; }

16

failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }

17

18

Usage() {

19

cat <<EOF

20

Usage: ${0##*/} [ options ] arch image

21

22

arch : one of i386 or x86_64

23

image : the image to upload and register

24

25

options:

26

-l|--add-launch <user_id> : user_id can be "all", or "none"

27

--bucket-mapper : relavent only with bucket-prefix

28

default: ${DEFAULT_MAPPER}

29

--bucket-prefix <pre> : bucket prefix to use

30

--buckets <list> : list of buckets to publish to

31

--dry-run : only report what would be done

32

--name name : register with '--name' to ec2-register

33

--path-prefix <pre> : path inside bucket to put image in

34

--resize size : resize the image with 'resize-cloud-image'

35

before publishing.

36

--regions <list> : list of regions to publish to

37

default: all (per describe-regions)

38

--allow-existing : if a image is already registered

39

simply report as if work was done

40

-o|--output <file> : write registered id and manifest to file

41

-r|--rename <publish_path> : publish to bucket/<publish_path>

42

default: bucket/<basename(image)>

43

-t|--type <type> : type is one of kernel/ramdisk/image

44

-v|--verbose : increase verbosity

45

46

all <list> entries are comma or space delimited, and map 1x1 with regions

47

EOF

48

}

49

50

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }

51

cleanup() {

52

local pids

53

pids=$(jobs -pr)

54

if [ -n "$pids" ]; then

55

error "waiting on pids: $pids"

56

kill $pids

57

wait $pids

58

fi

59

local x=""

60

for x in "$EXTRACT_D" "${RENAME_D}" "${TMPD}"; do

61

[ -z "${x}" -o ! -d "${x}" ] || rm -Rf "${x}"

62

done

63

64

return 0

65

}

66

67

debug() {

68

local level=${1}

69

shift;

70

[ "${level}" -gt "${VERBOSITY}" ] && return

71

error "$(date):" "${@}"

72

}

73

run() {

74

local dir="${1}" pre=${2} msg=${3};

75

shift 3;

76

[ -e "${dir}/stamp.${pre}" ] &&

77

{ debug 1 "skipping ${pre}"; return 0; }

78

debug 1 "${msg}"

79

echo "$@" > "${dir}/${pre}.cmd"

80

"$@" > "${dir}/${pre}.stdout" 2> "${dir}/${pre}.stderr" &&

81

: > "${dir}/stamp.${pre}" && return 0

82

local ret=$?

83

cat "${dir}/${pre}.stderr" 1>&2

84

return ${ret}

85

}

86

87

retry() {

88

local trycount=${1} sleep=${2}

89

shift; shift;

90

local i=0 smsg=" sleeping ${sleep}: $*" ret=0

91

for((i=0;i<${trycount};i++)); do

92

"$@" && return 0

93

ret=$?

94

[ $(($i+1)) -eq ${trycount} ] && smsg=""

95

debug 1 "Warning: cmd failed [try $(($i+1))/${trycount}].${smsg}"

96

sleep $sleep

97

done

98

return $ret

99

}

100

101

search_args() {

102

local x="" i=0 needle="$1"

103

shift;

104

for x in "${@}"; do

105

[ "${needle}" = "${x}" ] && { _RET=$i; return 0; }

106

i=$(($i+1))

107

done

108

return 1

109

}

110

111

append_location() {

112

local pre=${1} region=${2} rsuf=""

113

[ -n "${pre}" -a -n "${region}" ] || exit 1;

114

region2loc "${region}" &&

115

rsuf=-$(echo "${_RET}" | tr '[:upper:]' '[:lower:]') &&

116

echo "${pre}${rsuf}"

117

}

118

119

append_short() {

120

local pre=${1} region=${2}

121

[ -n "${pre}" -a -n "${region}" ] || exit 1;

122

local rsuf=-${region%%-*}

123

# if the prefix ends with the suffix already, dont add it

124

[ "${pre%${rsuf}}" != "${pre}" ] && { echo "${pre}"; return 0; }

125

echo "${pre}${rsuf}"

126

}

127

128

append_full() {

129

local pre=${1} region=${2}

130

[ -n "${pre}" -a -n "${region}" ] || exit 1;

131

# if the prefix ends with the region already, dont add it

132

[ "${pre%${region}}" != "${pre}" ] && { echo "${pre}"; return 0; }

133

echo "${pre}-${region}"

134

}

135

136

checkstatus() {

137

local x="" i=0

138

for x in "$@"; do

139

[ "$x" = "0" ] || i=$(($i+1))

140

done

141

return $i

142

}

143

144

get_regions() {

145

local regs=""

146

[ "${#REGIONS[@]}" -eq 0 ] || return 0

147

REGIONS=( $(ec2-list-all-regions) ) || return 1

148

}

149

150

region2loc() {

151

[ "${1}" = "us-east-1" ] && { _RET="US"; return; }

152

[ "${1}" = "eu-west-1" ] && { _RET="EU"; return; }

153

_RET=${1}; return;

154

}

155

156

157

explode() {

158

local delim="$1" str="${2}" limit="${3:-0}"

159

_RET=( );

160

local ostr=""

161

while [ ${#_RET[@]} -lt ${limit} -o ${limit} -eq 0 ]; do

162

ostr=${str}

163

str=${str#*[${delim}]}

164

[ ${limit} -eq $((${#_RET[@]}+1)) -o "${str}" = "${ostr}" ] &&

165

{ _RET[${#_RET[@]}]=${ostr}; return 0; }

166

_RET[${#_RET[@]}]=${ostr%?${str}}

167

done

168

return

169

}

170

split_input() {

171

[ -n "${1}" ] || { _RET=( ); return 0; }

172

explode ", " "${1}"

173

}

174

175

select_migrate_source() {

176

# select_migrate_source(target, available_regions)

177

local target=$1 i=""

178

shift;

179

#_RET=$1; return; # just return the first available

180

_RET="" # never migrate

181

}

182

183

short_opts="hl:no:r:t:vw:"

184

long_opts="add-launch:,allow-existing,buckets:,bucket-mapper:,bucket-prefix:,dry-run,help,name:,output:,path-prefix:,regions:,rename:,resize:,type:,verbose,working-dir:"

185

getopt_out=$(getopt --name "${0##*/}" \

186

--options "${short_opts}" --long "${long_opts}" -- "$@") &&

187

eval set -- "${getopt_out}" ||

188

bad_Usage

189

190

add_acl=""

191

allow_existing=0

192

arch=""

193

bucket_pre=""

194

buckets=( )

195

bucket_mapper=${DEFAULT_MAPPER}

196

dry_run=0

197

image=""

198

locations=( )

199

output=""

200

path_prefix=""

201

ramdisk=""

202

ramdisks=( )

203

regions=( )

204

rename=""

205

resize=""

206

name=""

207

tries=3;

208

wdir_in=""

209

210

while [ $# -ne 0 ]; do

211

cur=${1}; next=${2};

212

case "$cur" in

213

-d|--working-dir) wdir_in=${next}; shift;;

214

-h|--help) Usage; exit 0;;

215

-k|--kernels)

216

split_input "${next}"

217

kernels=( "${kernels[@]}" "${_RET[@]}" );

218

shift;;

219

-l|--add-launch)

220

if [ "${next}" = "none" ]; then

221

add_acl=""

222

else

223

user=${next//-/}; # just be nice and remove '-'

224

add_acl="${add_acl:+${add_acl} }${user}";

225

fi

226

shift;;

227

--name) name=${next}; shift;;

228

-o|--output) output="${next}"; shift;;

229

-i|--ramdisks)

230

split_input "${next}"

231

ramdisks=( "${ramdisks[@]}" "${_RET[@]}" );

232

shift;;

233

-n|--dry-run) dry_run=1;;

234

-r|--rename) rename=${next}; shift;;

235

-v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;

236

--allow-existing) allow_existing=1;;

237

--bucket-mapper) bucket_mapper=${next}; shift;;

238

--bucket-prefix) bucket_pre=${next}; shift;;

239

--buckets)

240

split_input "${next}"

241

buckets=( "${buckets[@]}" "${_RET[@]}" );

242

shift;;

243

--path-prefix) path_prefix=${next}; shift;;

244

--regions)

245

if [ "${next}" = "all" ]; then

246

get_regions || fail "failed to get region list"

247

regions=( "${REGIONS[@]}" );

248

else

249

split_input "${next}"

250

regions=( "${regions[@]}" "${_RET[@]}" );

251

fi

252

shift;;

253

--resize) resize=$next; shift;;

254

--) shift; break;;

255

-*) bad_Usage "confused by ${cur}";;

256

esac

257

shift;

258

done

259

260

[ $# -lt 2 ] && bad_Usage "must provide arch, image"

261

[ $# -gt 2 ] && bad_Usage "unexpected arguments: ${3}"

262

arch="${1}"

263

image="${2}"

264

image_in="$image"

265

266

# Make sure that non-amd64 builds don't happen

267

[ "${arch}" == "amd64" ] || {

268

debug 1 "HVM instance-store publication is only supported on amd64, not ${arch}";

269

exit 0;

270

}

271

272

[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }

273

274

[ "${arch}" = "amd64" ] && arch=x86_64 && debarch=amd64 || debarch=${arch}

275

276

[ "$arch" = "i386" -o "$arch" = "x86_64" ] ||

277

bad_Usage "arch must be i386 or x86_64"

278

279

[ -n "${bucket_pre}" -a "${#buckets[@]}" -ne 0 ] &&

280

fail "bucket-prefix and bucket list are incompatible"

281

282

get_regions || fail "failed to get region list"

283

284

[ "${#regions[@]}" -eq 0 ] && regions=( "${REGIONS[@]}" )

285

286

for region in "${regions[@]}"; do

287

search_args "${region}" "${REGIONS[@]}" ||

288

bad_Usage "region ${region} is not valid"

289

290

region2loc "${region}" ||

291

bad_Usage "unable to map ${region} to a location"

292

locations[${#locations[@]}]=${_RET}

293

294

if [ -n "${bucket_pre}" ]; then

295

bucket=$(${bucket_mapper} "${bucket_pre}" "${region}" "${arch}") ||

296

fail "unable to map bucket name for ${region} (${bucket_mapper})"

297

buckets[${#buckets[@]}]=${bucket}

298

fi

299

done

300

301

[ ${#kernels[@]} -eq 0 -o ${#kernels[@]} -eq ${#regions[@]} ] ||

302

fail "must provide kernel for each region"

303

304

[ ${#ramdisks[@]} -eq 0 -o ${#ramdisks[@]} -eq ${#regions[@]} ] ||

305

fail "must provide ramdisk for each region"

306

307

[ "${#regions[@]}" -eq "${#buckets[@]}" ] ||

308

failp "%s\n\t%s\n" "number of regions != number of buckets" \

309

"see --buckets or --bucket-prefix"

310

311

[ -n "${rename}" ] || rename=${image##*/}

312

manifest_path="${path_prefix:+${path_prefix%/}/}${rename}.manifest.xml"

313

314

if [ -n "${wdir_in}" ]; then

315

[ -d "${wdir_in}" ] || fail "input working directory not a directory";

316

wdir=$(readlink -f "${wdir_in}") ||

317

fail "failed to realize ${wdir_in}"

318

else

319

TMPD=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX) ||

320

fail "failed to make tmpdir"

321

wdir="${TMPD}"

322

trap cleanup EXIT

323

fi

324

325

if [ -e "${wdir}/${rename}" ]; then

326

[ "${wdir}/${rename}" -ef "${img}" ] ||

327

fail "${wdir} already contains file named ${rename}"

328

fi

329

330

# if name is not given, then create one, use a shortened

331

# manifest name. if bucket_pre is given, then use that as part

332

# of the name.

333

if [ -z "${name}" ]; then

334

# there is no longer support for tools without '--name'

335

name=${rename%.manifest.xml}; name=${name%.img};

336

if [ -n "${bucket_pre}" ]; then

337

name="${bucket_pre}/${path_prefix:+${path_prefix%/}/}${name}"

338

fi

339

register_name=( --name "${name}" )

340

fi

341

342

mf_bucket=""

343

to_publish=( )

344

registered_ids=( )

345

for((i=0;i<${#regions[@]};i++)); do

346

region=${regions[$i]}

347

bucket=${buckets[$i]}

348

349

# hack! instance store and kernels renamed:

350

# old 099720109477/ubuntu-images-milestone/ubuntu-lucid-10.04-beta1-amd64-server-20100317

351

# new 099720109477/ubuntu/images-milestone/ubuntu-lucid-10.04-beta1-amd64-server-20100317

352

# so we check for 'ubuntu.images' rather than what was existing

353

exist_reg="/${name//ubuntu\//ubuntu.}$"

354

debug 1 "checking for existing registered image $region at ${exist_reg}"

355

tmp=$(xc2 ximages matching --region "$region" \

356

"name ~ /${exist_reg//\//\\/}/ && name !~ /\/ebs\//" -o self) ||

357

fail "failed to check for existing manifest in $region"

358

359

if [ -n "${tmp}" ]; then

360

tmp=$(echo "$tmp" | awk '-F\t' '{print $2, $3; exit(0); }')

361

set -- ${tmp}

362

img_id=${1}; path=${2}

363

[ ${allow_existing} -eq 1 ] ||

364

fail "$name already registered as ${img_id} ($path)"

365

debug 1 "using existing ${img_id} for ${bucket}/${manifest_path}"

366

367

# add this to a migrate from candidates

368

mf_regions[${#mf_regions[@]}]=$region;

369

registered_ids[$i]=${img_id}

370

else

371

to_publish[${#to_publish[@]}]=$i

372

debug 1 "no existing registered image in ${regions[i]}"

373

fi

374

done

375

376

# if any of the regions were not going to be migrate, then

377

# migrate-image only works for image, not kernel/initrd (LP: #428692)

378

need_image=0

379

for i in "${to_publish[@]}"; do

380

region=${regions[$i]}

381

select_migrate_source "$region" "${mf_regions[@]}"

382

if [ -n "$_RET" -a "$img_type" = "image" ]; then

383

debug 1 "publish to $region, migrating from $_RET"

384

else

385

need_image=1

386

fi

387

done

388

389

if [ $need_image -ne 0 ]; then

390

pub_image="${TMPD}/${image##*/}"

391

new_image="${pub_image//.img/.raw}"

392

cp ${image} ${pub_image} ||

393

fail "failed to copy image for manipulation"

394

395

if [ -n "$resize" ]; then

396

# resize the image to 10G

397

debug 1 "resizing image to $resize"

398

qemu-img resize "${pub_image}" "$resize" 1>&2 ||

399

fail "failed to resize ${image}"

400

fi

401

402

debug 1 "converting qcow image to raw"

403

qemu-img convert -O raw "${pub_image}" "${new_image}" ||

404

fail "failed to convert image"

405

image="${new_image}"

406

407

if [ -n "${resize}" ]; then

408

growpart "${image}" 1 ||

409

fail "failed to grow partition images"

410

fi

411

412

RENAME_D=$(mktemp -d "${wdir}/.rename.XXXXXX") &&

413

rename_full="${RENAME_D}/${rename}" &&

414

mkdir -p $(dirname ${rename_full}) &&

415

ln -sf "${image}" "${RENAME_D}/${rename}" ||

416

fail "symlink failed: working-dir/rename/${rename} -> ${image_full}"

417

418

bundle_cmd="bundle-image"

419

bundle_args=( "--image" "${rename_full}" )

420

421

run "${wdir}" "bundle.once" "bundling hvm-image ${image}" \

422

xc2 "${bundle_cmd}" --destination "${wdir}" --arch "${arch}" \

423

"${bundle_args[@]}" ||

424

fail "failed to bundle hvm image"

425

fi

426

427

manifest="${rename##*/}.manifest.xml"

428

if [ "${#to_publish[@]}" -ne 0 ]; then

429

for i in "${to_publish[@]}"; do

430

region=${regions[$i]}

431

location=${locations[$i]}

432

bucket=${buckets[$i]}

433

kernel=${kernels[$i]}

434

ramdisk=${ramdisks[$i]}

435

bucketpath="${bucket}${path_prefix:+/${path_prefix%/}}"

436

437

# this would select a good source region for a migrate to region

438

select_migrate_source "$region" "${mf_regions[@]}"

439

if [ -n "$_RET" ]; then

440

mf_region="$_RET"

441

search_args "$mf_region" "${regions[@]}"

442

mf_bucket=${buckets[$_RET]}

443

[ -n "$mf_bucket" ] || fail "did not find $region in $regions"

444

fi

445

446

bucketpath="${bucket}${path_prefix:+/${path_prefix%/}}"

447

( # start a subshell so we can run each region parallel

448

449

retry 10 10s \

450

xc2 create-bucket "$bucket" "$location" >/dev/null ||

451

{ error "failed to create bucket"; exit 1; }

452

453

# note, as found in LP: #428692 comment 10, migrate-image doesnt work

454

# for kernel/initrd. So, use publish

455

set -x

456

if [ -z "$mf_bucket" -o "$img_type" != "image" ]; then

457

debug 2 xc2 upload-bundle --bucket "${bucketpath}" \

458

--manifest "${wdir}/${manifest}" --retry

459

retry 20 1m \

460

run "${wdir}" "upload.${region}" \

461

"upload ${bucket}/${manifest_path} to ($region)" \

462

xc2 upload-bundle --bucket "${bucketpath}" \

463

--manifest "${wdir}/${manifest}" --retry || {

464

error "failed to upload bundle to ${bucket}/${manifest_path}";

465

exit 1;

466

}

467

else

468

debug 2 xc2 migrate-image --bucket "${mf_bucket}" \

469

--manifest "${manifest_path}" \

470

--destination-bucket "${bucket}" \

471

--region "${region}" ||

472

retry 20 1m \

473

run "${wdir}" "migrate.${region}" \

474

"migrating ${img_type} ${manifest_path} (${mf_bucket}=>${region})" \

475

xc2 migrate-image --bucket "${mf_bucket}" \

476

--manifest "${manifest_path}" \

477

--destination-bucket "${bucket}" \

478

--region "${region}" ||

479

{ error "failed to migrate to ${region}"; exit 1; }

480

fi

481

junk="" img_id=""; set -x

482

run "${wdir}" "register.${region}" "register ${bucket}/${manifest_path}" \

483

xc2 ximages register --region "${region}" --headers --debug \

484

--architecture "${arch}" --virtualization-type=hvm \

485

"${bucket}/${manifest_path}" \

486

"${register_name[@]}" \

487

"${krd_args[@]}" && \

488

read junk img_id < "${wdir}/register.${region}.stdout" &&

489

[ "${img_id#???-}" != "${img_id}" ] || {

490

# Warning: register with an invalid kernel or ramdisk

491

# can register the image (per describe-images) even though

492

# command ec2-register command will fail.

493

# also, it does not write the new ami to stdout

494

if bad=$(xc2 get-manifest-id --region "$region" \

495

"${bucket}/${manifest_path}") &&

496

[ -n "${bad}" ]; then

497

set -- ${bad}

498

bad_id=${1}

499

error "un-registering invalid $bad"

500

xc2 deregister --region "$region" "${bad_id}"

501

fi

502

errorp "failed to register ${bucket}/${manifest_path}.%s\n" \

503

"${junk:+ output:${junk} ${img_id}}"

504

exit 1;

505

}

506

507

debug 1 "registered at ${region}: ${bucket}/${manifest_path} as ${img_id}"

508

) &

509

510

PUBLISHES[$i]=$!

511

done

512

513

for i in "${to_publish[@]}"; do

514

region=${regions[$i]}

515

wait "${PUBLISHES[$i]}" || fail "failed to publish to ${regions[$i]}"

516

read junk img_id < "$wdir/register.$region.stdout"

517

registered_ids[$i]=$img_id

518

done

519

fi

520

521

label_type="hvm-instance-store"

522

for((i=0;i<${#regions[@]};i++)); do

523

img_id=${registered_ids[$i]}

524

bucket=${buckets[$i]}

525

region=${regions[$i]}

526

if [ -z "${output}" -o "${output}" = "-" ]; then

527

printf "%s\t%s\t%s\t%s\t%s\n" "${region}" "${img_id}" "${debarch}" "${label_type}" "${bucket}/${manifest}"

528

else

529

printf "%s\t%s\t%s\t%s\t%s\n" "${region}" "${img_id}" "${debarch}" "${label_type}" "${bucket}/${manifest}" >> "${output}"

530

fi

531

done

532

533

# now add acls for everything

534

for((i=0;i<${#registered_ids[@]};i++)); do

535

img_id=${registered_ids[$i]}

536

region=${reg_regions[$i]}

537

for user in ${add_acl}; do

538

run "${wdir}" "add_user.${region}.${user}" \

539

"add ${user} to ${manifest} in ${region}" \

540

xc2 euca modify-image-attribute --region "${region}" \

541

--launch-permission --add "${user}" "${img_id}" ||

542

fail "failed to add launch permission for ${user} to ${img_id}"

543

done

544

done

545

546

exit 0