~davewalker/ubuntu-on-ec2/cloud-utils : revision 58

20

# along with this program. If not, see <http://www.gnu.org/licenses/>.

21

22

23

import os, string, sys

24

25

def usage():

26

usage = "\n\

27

Usage:\n\

28

uec-run-instances [-l|--launchpad-id lp_id_1,lp_id_2,lp_id_3]\n\

29

[euca-run-instances options]\n\

30

\n\

31

This program is a wrapper script for euca-run-instances(1) that takes one\n\

32

additional option, -l|--launchpad-id. With this option, a user can\n\

33

specify a comma-separated list of Launchpad.net usernames. Once the\n\

34

instance is booted, the cloud-init boot script will retrieve the\n\

35

public ssh keys of the specified users from Launchpad.net using\n\

36

ssh-import-lp-id(1).\n\

37

\n\

38

All other options besides [-l|--launchpad-id] are simply passed\n\

39

on to euca-run-instances(1).\n"

40

print(usage)

23

import os

24

import string

25

import sys

26

import signal

27

import re

28

import base64

29

from optparse import OptionParser

30

from socket import getaddrinfo

31

import time

32

import logging

33

from paramiko import SSHClient, AutoAddPolicy, AuthenticationException

34

import paramiko

35

from subprocess import Popen, PIPE

36

37

finished = "FINISHED"

38

39

class SafeConnectException(Exception):

40

pass

41

42

class Instance(object):

43

pass

44

45

class TemporaryMissingHostKeyPolicy(AutoAddPolicy):

46

""" does not save to known_hosts, but does save the keys in an array """

47

def __init__(self):

48

self._keys = []

49

AutoAddPolicy.__init__(self)

50

51

def missing_host_key(self, client, hostname, key):

52

self._keys.append(key)

53

54

def getKeys(self):

55

return self._keys

56

57

class PermanentMissingHostKeyPolicy(TemporaryMissingHostKeyPolicy):

58

""" also has the behavor of the parent AutoAddPolicy """

59

def missing_host_key(self, client, hostname, key):

60

#TemporaryMissingHostKeyPolicy.missing_host_key(self, client, hostname, key)

61

self._keys.append(key)

62

AutoAddPolicy.missing_host_key(self, client, hostname, key)

63

64

class ConsoleFingerprintScanner(object):

65

def __init__(self, instance_id, hostname, provider, options, sleeptime=30):

66

self.state = "working"

67

self.instance_id = instance_id

68

self.hostname = hostname

69

self.provider = provider

70

self.sleeptime = sleeptime

71

self.fingerprint = None

72

self.options = options

73

self.logger = logging.getLogger('console-scanner(%s)' % instance_id)

74

75

def scan(self):

76

self.logger.debug('scraping fingerprints for instance_id = %s', self.instance_id)

77

try:

78

while self.fingerprint is None:

79

console_data = self.get_console_output()

80

self.fingerprint = self.get_fingerprints_in_console_data(console_data)

81

if self.fingerprint is not None:

82

self.fingerprint = (int(self.fingerprint[0]), self.fingerprint[1], self.fingerprint[3])

83

else:

84

self.logger.debug('sleeping %d seconds', self.options.sleep_time)

85

time.sleep(self.options.sleep_time)

86

except None:

87

pass

88

return self.fingerprint

89

90

def get_console_output(self):

91

cmd = '%s-get-console-output' % self.provider

92

args = [ cmd ]

93

args.append(self.instance_id)

94

95

self.logger.debug('running %s', args)

96

rconsole = Popen(args, stdout=PIPE)

97

98

ret = []

99

try:

100

for line in rconsole.stdout:

101

ret.append(line.strip())

102

finally:

103

cmdout = rconsole.wait()

104

105

if bool(cmdout):

106

raise Exception('%s failed with return code = %d', cmd, cmdout)

107

108

return ret

109

110

def get_fingerprints_in_console_data(self, output):

111

# return an empty list on "no keys found"

112

# return a list of key fingerprint data on success

113

# where each key fingerprint data is an array like:

114

# (2048 c7:c8:1d:0f:d9:40:20:f9:47:6a:f9:1d:6f:0a:8a:fe localhost (RSA))

115

begin_marker="-----BEGIN SSH HOST KEY FINGERPRINTS----"

116

end_marker="----END SSH HOST KEY FINGERPRINTS-----"

117

i = 0

118

while i < len(output):

119

if output[i].find(begin_marker) > -1:

120

while i < len(output) and output[i].find(end_marker) == -1:

121

self.logger.debug(output[i].strip())

122

toks = output[i].split(" ")

123

self.logger.debug(toks)

124

if len(toks) == 5: toks=toks[1:] # rip off "ec2:"

125

if len(toks) == 4 and toks[3] == "(RSA)":

126

self.logger.debug('found %s on line %d', toks, i)

127

return((toks))

128

i=i+1

129

break

130

i=i+1

131

self.logger.debug('did not find any fingerprints in output! (lines=%d)', i)

132

return None

133

134

class SshKeyScanner(object):

135

def __init__(self, instance_id, hostname, options, sleeptime=30):

136

self.state = "working"

137

self.instance_id = instance_id

138

self.hostname = hostname

139

self.sleeptime = sleeptime

140

self.fingerprint = None

141

self.keys = None

142

self.options = options

143

self.port = 22

144

self.logger = logging.getLogger('ssh-key-scanner(%s)' % instance_id)

145

self.client = None

146

self.connected = False

147

148

def scan(self):

149

self.logger.debug('getting fingerprints for %s', self.hostname)

150

try:

151

fingerprints = self.get_fingerprints_for_host()

152

self.logger.debug('fingerprints = %s', fingerprints)

153

if(len(fingerprints)>0):

154

self.state = "finished"

155

self.fingerprint = fingerprints[0]

156

except None:

157

pass

158

return self.fingerprint

159

160

def get_fingerprints_for_host(self):

161

# return an empty list on "no keys found"

162

# return a list of key fingerprint data on success

163

# where each key fingerprint data is an array like:

164

# (2048 c7:c8:1d:0f:d9:40:20:f9:47:6a:f9:1d:6f:0a:8a:fe localhost (RSA))

165

166

# use paramiko here

167

self.client = SSHClient()

168

client = self.client

169

client.set_log_channel('ssh-key-scanner(%s)' % self.instance_id)

170

171

if self.options.known_hosts is not None:

172

policy = PermanentMissingHostKeyPolicy()

173

""" This step ensures we save the keys, otherwise that step will be

174

skipped in AutoAddPolicy.missing_host_key """

175

for path in self.options.known_hosts:

176

if not os.path.isfile(path):

177

# if the file doesn't exist, then

178

# create it empty

179

fp = open(path,"w")

180

fp.close()

181

client.load_host_keys(path)

182

else:

183

policy = TemporaryMissingHostKeyPolicy()

184

client.set_missing_host_key_policy(policy)

185

186

pkey = None

187

if self.options.privkey is not None:

188

# TODO support password protected key file

189

pkey = paramiko.RSAKey.from_private_key_file(self.options.privkey)

190

191

retries = 0

192

193

allkeys = []

194

195

while 1:

196

try:

197

client.connect(self.hostname,self.port,username=self.options.ssh_user,pkey=pkey)

198

self.connected = True

199

break

200

except AuthenticationException as (message):

201

self.logger.warning('auth failed (non fatal) %s', message)

202

break

203

except Exception as (e):

204

retries += 1

205

if retries > 5:

206

raise Exception('gave up after retrying ssh %d times' % retries)

207

self.logger.info(e)

208

self.logger.debug('retry #%d... sleeping %d seconds..', retries, self.options.sleep_time)

209

time.sleep(self.options.sleep_time)

210

211

212

rlist = []

213

214

allkeys.extend(policy.getKeys())

215

allkeys.append(client.get_transport().get_remote_server_key())

216

217

for key in allkeys:

218

219

if type(key) == paramiko.RSAKey or type(key) == paramiko.PKey:

220

keytype = '(RSA)'

221

elif type(key) == paramiko.DSSKey:

222

keytype = '(DSA)'

223

else:

224

raise Exception('Cannot handle type %s == %s' % (type(key).__name__, key))

225

226

fp = key.get_fingerprint().encode("hex")

227

fp = ':'.join(re.findall('..', fp))

228

rlist.append((key.get_bits(), fp, keytype))

229

230

return rlist

231

232

def run_commands(self):

233

if self.options.ssh_run_cmd is not None and len(self.options.ssh_run_cmd):

234

if not self.connected:

235

self.logger.critical('cannot run command, ssh did not connect')

236

sys.exit(1)

237

ecmd = ' '.join(self.options.ssh_run_cmd)

238

self.logger.debug('running %s', ecmd)

239

inouterr = self.client.exec_command(ecmd)

240

try:

241

for line in inouterr[1]:

242

print line,

243

except:

244

pass

245

try:

246

for line in inouterr[2]:

247

print >> sys.stderr(line)

248

except:

249

pass

250

251

if self.connected:

252

self.client.close()

253

self.connected = False

254

255

def get_auto_instance_type(ami_id, provider):

256

cmd = '%s-describe-images' % provider

257

args = [ cmd , ami_id ]

258

logging.debug('running %s', args)

259

rimages = Popen(args, stdout=PIPE)

260

deftype = { 'i386' : 'm1.small', 'x86_64' : 'm1.large' }

261

262

try:

263

for line in rimages.stdout:

264

# Just in case there are %'s, don't confusee logging

265

# XXX print these out instead

266

logging.debug(line.replace('%','%%').strip())

267

parts = line.split("\t")

268

if parts[0] == 'IMAGE':

269

itype = parts[7]

270

if itype in deftype:

271

logging.info('auto instance type = %s', deftype[itype])

272

return deftype[itype]

273

finally:

274

rcode = rimages.wait()

275

276

logging.warning('ami not found, returning default m1.small')

277

return("m1.small")

278

279

def timeout_handler(signum, frame):

280

logging.critical('timeout reached, exiting')

41

281

sys.exit(1)

42

282

43

if len(sys.argv) < 2:

44

usage()

45

46

lp_ids = ""

47

user_data_str = ""

48

i = 0

49

args = sys.argv

50

del(args[i])

51

while i < len(args):

52

if args[i] == "-l" or args[i] == "--launchpad-id":

53

del(args[i])

54

lp_ids = args[i]

55

del(args[i])

56

i += 1

57

58

if len(lp_ids) > 0:

59

i = 1

60

while i < len(args):

61

if args[i] == "-d" or args[i] == "--user-data" or args[i] == "-f" or args[i] == "--user-data-file":

62

print("ERROR: User data is not supported with the -l|--launchpad-id option")

63

sys.exit(1)

64

i += 1

65

lp_ids = string.replace(lp_ids, ",", " ")

66

args.insert(0, "#cloud-config\nruncmd:\n - sudo -Hu ubuntu ssh-import-lp-id %s" % lp_ids)

67

args.insert(0, "-d")

68

69

args.insert(0, "euca-run-instances")

70

os.execvp("euca-run-instances", args)

283

def handle_runargs(option, opt_str, value, parser):

284

delim=getattr(parser.values,"runargs_delim",None)

285

cur=getattr(parser.values,"runargs",[])

286

if cur is None: cur = []

287

cur.extend(value.split(delim))

288

setattr(parser.values,"runargs",cur)

289

return

290

291

def main():

292

parser = OptionParser(usage="usage: %prog [options] ids|(-- raw args for provider scripts)")

293

parser.add_option("-t", "--instance-type", dest="inst_type",

294

help="instance type", metavar="TYPE",

295

default="auto")

296

parser.add_option("-n", "--instance-count", dest="count",

297

help="instance count", metavar="TYPE", type="int",

298

default=1)

299

parser.add_option("", "--ssh-privkey", dest="privkey",

300

help="private key to connect with (ssh -i)", metavar="id_rsa",

301

default=None)

302

parser.add_option("", "--ssh-pubkey", dest="pubkey",

303

help="public key to insert into image)", metavar="id_rsa.pub",

304

default=None)

305

parser.add_option("", "--ssh-run-cmd", dest="ssh_run_cmd", action="append", nargs=0,

306

help="run this command when ssh'ing", default=None)

307

parser.add_option("","--ssh-user", dest="ssh_user",

308

help="connect with ssh as user", default=None)

309

parser.add_option("", "--associate-ip", dest="ip",

310

help="associate elastic IP with instance", metavar="IP_ADDR",

311

default=None)

312

parser.add_option("", "--known-hosts", dest="known_hosts", action="append",

313

help="write host keys to specified known_hosts file. Specify multiple times to read keys from multiple files (only updates last one)",

314

metavar="KnownHosts", default=None)

315

parser.add_option("-l", "--launchpad-id", dest="launchpad_id", action="append",

316

help="launchpad ids to pull SSH keys from (multiple times adds to the list)",

317

metavar="lpid", default=None)

318

parser.add_option("-i", "--instance-ids", dest="instance_ids", action="store_true",

319

help="expect instance ids instead of ami ids, skips -run-instances", default=False)

320

parser.add_option("", "--all-instances", dest="all_instances", action="store_true",

321

help="query all instances already defined (running/pending/terminated/etc)", default=False)

322

parser.add_option("","--run-args", dest="runargs", action="callback",

323

callback=handle_runargs, type="string",

324

help="pass option through to run-instances")

325

parser.add_option("","--run-args-delim", dest="runargs_delim",

326

help="split run-args options with delimiter",

327

default=None)

328

parser.add_option("","--verify-ssh", dest="verify_ssh", action="store_true",

329

help="verify SSH keys against console output (implies --wait-for=ssh)",

330

default=False)

331

parser.add_option("","--wait-for", dest="wait_for",

332

help="wait for one of: ssh , running",default=None)

333

parser.add_option("-p","--provider", dest="provider",

334

help="either euca or ec2", default=None)

335

parser.add_option("-v", "--verbose", action="count", dest="loglevel",

336

help="increase logging level", default=3)

337

parser.add_option("-q", "--quiet", action="store_true", dest="quiet",

338

help="produce no output or error messages", default=False)

339

parser.add_option("", "--sleep-time", dest="sleep_time",

340

help="seconds to sleep between polling", default=2)

341

parser.add_option("", "--teardown", dest="teardown", action="store_true",

342

help="terminate instances at the end", default=False)

343

344

(options, args) = parser.parse_args()

345

346

if len(args) < 1 and not options.all_instances:

347

parser.error('you must pass at least one ami ID')

348

349

# loglevel should be *reduced* every time -v is passed, see logging docs for more

350

if options.quiet:

351

sys.stderr = open('/dev/null','w')

352

sys.stdout = sys.stderr

353

else:

354

loglevel = 6 - options.loglevel

355

if loglevel < 1:

356

loglevel = 1

357

# logging module levels are 0,10,20,30 ...

358

loglevel = loglevel * 10

359

360

logging.basicConfig(level=loglevel,format="%(asctime)s %(name)s/%(levelname)s: %(message)s",stream=sys.stderr)

361

362

logging.debug("loglevel = %d", loglevel)

363

364

provider = options.provider

365

if options.provider is None:

366

provider = os.getenv('EC2PRE','euca')

367

368

if options.ssh_run_cmd == [()]:

369

options.ssh_run_cmd = args

370

371

if options.known_hosts is None:

372

options.known_hosts = [ os.path.expanduser('~/.ssh/known_hosts') ]

373

374

if options.known_hosts is not None and len(options.known_hosts):

375

path = None

376

for path in options.known_hosts:

377

if not os.access(path, os.R_OK):

378

logging.warning('known_hosts file %s is not readable!', path)

379

# paramiko writes to the last one

380

if not os.access(path, os.W_OK):

381

logging.critical('known_hosts file %s is not writable!', path)

382

383

logging.debug("provider = %s", provider)

384

385

logging.debug("instance type is %s", options.inst_type)

386

387

if options.instance_ids or options.all_instances:

388

389

if options.all_instances:

390

pending_instance_ids = ['']

391

else:

392

pending_instance_ids = args

393

394

else:

395

396

if len(args) < 1:

397

raise Exception('you must pass at least one AMI ID')

398

399

ami_id = args[0]

400

del(args[0])

401

402

logging.debug("ami_id = %s", ami_id)

403

404

if options.inst_type == "auto":

405

options.inst_type = get_auto_instance_type(ami_id, provider)

406

407

pending_instance_ids = []

408

409

cmd = '%s-run-instances' % provider

410

411

run_inst_args = [ cmd ]

412

413

# these variables pass through to run-instances

414

run_inst_pt = {

415

"instance-count" : options.count,

416

"instance-type" : options.inst_type,

417

}

418

419

for key, val in run_inst_pt.iteritems():

420

if key is not None and key != "":

421

run_inst_args.append("--%s=%s" % (key,val))

422

423

if options.launchpad_id:

424

run_inst_args.append('-d')

425

run_inst_args.append(

426

"#cloud-config\nruncmd:\n - sudo -Hu ubuntu ssh-import-lp-id " + ','.join(options.launchpad_id))

427

428

if options.runargs is not None:

429

run_inst_args.extend(options.runargs)

430

431

run_inst_args.append(ami_id)

432

433

# run-instances with pass through args

434

logging.debug("executing %s",run_inst_args)

435

logging.info("starting instances with ami_id = %s", ami_id)

436

437

rinstances = Popen(run_inst_args, stdout=PIPE)

438

#INSTANCE i-32697259 ami-2d4aa444 pending 0 m1.small 2010-06-18T18:28:21+0000 us-east-1b aki-754aa41c monitoring-disabled instance-store

439

try:

440

for line in rinstances.stdout:

441

# Just in case there are %'s, don't confusee logging

442

# XXX print these out instead

443

logging.debug(line.replace('%','%%').strip())

444

parts = line.split("\t")

445

if parts[0] == 'INSTANCE':

446

pending_instance_ids.append(parts[1])

447

finally:

448

rcode = rinstances.wait()

449

450

logging.debug("command returned %d", rcode)

451

logging.info("instances started: %s", pending_instance_ids)

452

453

if bool(rcode):

454

raise Exception('%s failed' % cmd)

455

456

if len(pending_instance_ids) < 1:

457

raise Exception('no instances were started!')

458

459

cmd = '%s-describe-instances' % provider

460

461

instances = []

462

463

timeout_date = time.time() + 600

464

465

signal.signal(signal.SIGALRM, timeout_handler)

466

signal.alarm(600)

467

468

logging.debug("timeout at %s", time.ctime(timeout_date))

469

470

# We must wait for ssh to run commands

471

if options.verify_ssh and not options.wait_for == 'ssh':

472

logging.info('--verify-ssh implies --wait-for=ssh')

473

options.wait_for = 'ssh'

474

475

if options.ssh_run_cmd and not options.wait_for == 'ssh':

476

logging.info('--ssh-run-cmd implies --wait-for=ssh')

477

options.wait_for = 'ssh'

478

479

while len(pending_instance_ids):

480

new_pending_instance_ids = []

481

describe_inst_args = [ cmd ]

482

483

# remove '', confuses underlying commands

484

pids = []

485

for iid in pending_instance_ids:

486

if len(iid):

487

pids.append(iid)

488

if len(pids):

489

describe_inst_args.extend(pending_instance_ids)

490

491

logging.debug('running %s',describe_inst_args)

492

rdescribe = Popen(describe_inst_args, stdout=PIPE)

493

try:

494

for line in rdescribe.stdout:

495

logging.debug(line.replace('%','%%').strip())

496

parts = line.split("\t")

497

if parts[0] == 'INSTANCE':

498

iid = parts[1]

499

istatus = parts[5]

500

if istatus == 'terminated':

501

logging.debug('%s is terminated, ignoring...', iid)

502

elif istatus != 'running' and options.wait_for:

503

logging.warning('%s is %s', iid, istatus)

504

new_pending_instance_ids.append(iid)

505

else:

506

logging.info("%s %s", iid, istatus)

507

inst = Instance()

508

inst.id = iid

509

inst.hostname = parts[3]

510

inst.output = line

511

instances.append(inst)

512

finally:

513

rcode = rdescribe.wait()

514

515

pending_instance_ids = new_pending_instance_ids

516

517

logging.debug("command returned %d", rcode)

518

logging.debug("pending instances: %s", pending_instance_ids)

519

520

if bool(rcode):

521

raise Exception('%s failed' % cmd)

522

523

if len(pending_instance_ids):

524

logging.debug('sleeping %d seconds', options.sleep_time)

525

time.sleep(options.sleep_time)

526

527

if options.ip:

528

ips = options.ip.split(',')

529

if len(ips) < len(instances):

530

logging.warning('only %d ips given, some instances will not get an ip', len(ips))

531

elif len(ips) > len(instances):

532

logging.warning('%d ips given, some ips will not be associated', len(ips))

533

534

rcmds = []

535

ips.reverse()

536

for inst in instances:

537

cmd = '%s-associate-address' % provider

538

if len(ips) < 1:

539

break

540

ip = ips.pop()

541

aargs = [ cmd , '-i', inst.id, ip ]

542

logging.debug('running %s',aargs)

543

rassociate = Popen(aargs, stdout=PIPE)

544

rcmds.append(rassociate)

545

for rcmd in rcmds:

546

# dump stdin into the inst object

547

try:

548

for line in rcmd.stdout:

549

logging.info(line)

550

finally:

551

ret = rcmd.wait()

552

if bool(ret):

553

logging.debug('associate-ip returned %d', ret)

554

555

if options.wait_for == 'ssh':

556

logging.info('waiting for ssh access')

557

for inst in instances:

558

pid = os.fork()

559

if pid == 0:

560

ssh_key_scan = SshKeyScanner(inst.id, inst.hostname, options)

561

ssh_fingerprint = ssh_key_scan.scan()

562

if options.verify_ssh:

563

# For ec2, it can take 3.5 minutes or more to get console output, do this last, and only if

564

# we have to.

565

cons_fp_scan = ConsoleFingerprintScanner(inst.id, inst.hostname, provider, options)

566

console_fingerprint = cons_fp_scan.scan()

567

568

if console_fingerprint == ssh_fingerprint:

569

logging.debug('fingerprint match made for iid = %s',inst.id)

570

else:

571

raise Exception('fingerprints do not match for iid = %s' % inst.id)

572

ssh_key_scan.run_commands()

573

raise SystemExit

574

else:

575

logging.debug('child pid for %s is %d', inst.id, pid)

576

inst.child = pid

577

logging.info('Waiting for %d children', len(instances))

578

final_instances = []

579

580

for inst in instances:

581

try:

582

(pid, status) = os.waitpid(inst.child, 0)

583

except:

584

logging.critical('%s - %d doesn\'t exist anymore?', inst.id, pid)

585

logging.debug('%d returned status %d', pid, status)

586

if not bool(status):

587

final_instances.append(inst)

588

instances = final_instances

589

590

""" If we reach here, all has happened in the expected manner so

591

we should produce the expected output which is instance-id\\tip\\n """

592

593

final_instance_ids = []

594

for inst in instances:

595

final_instance_ids.append(inst.id)

596

597

if options.teardown:

598

terminate = [ '%s-terminate-instances' % provider ]

599

terminate.extend(final_instance_ids)

600

logging.debug('running %s', terminate)

601

logging.info('terminating instances...')

602

rterm = Popen(terminate,stdout=sys.stderr,stderr=sys.stderr)

603

rterm.wait()

604

605

if __name__ == "__main__":

606

main()