~anton-skriptsov/charms/trusty/cinder-nexentaedge/trunk : revision 21

1

2

#

3

# This file is part of charm-helpers.

4

#

5

# charm-helpers is free software: you can redistribute it and/or modify

6

# it under the terms of the GNU Lesser General Public License version 3 as

7

# published by the Free Software Foundation.

8

#

9

# charm-helpers is distributed in the hope that it will be useful,

10

# but WITHOUT ANY WARRANTY; without even the implied warranty of

11

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12

# GNU Lesser General Public License for more details.

13

#

14

# You should have received a copy of the GNU Lesser General Public License

15

# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.

16

17

import io

18

import json

19

import logging

20

import os

21

import re

22

import socket

23

import subprocess

24

import sys

25

import time

26

import uuid

27

28

import amulet

29

import distro_info

30

import six

31

from six.moves import configparser

32

if six.PY3:

33

from urllib import parse as urlparse

34

else:

35

import urlparse

36

37

38

class AmuletUtils(object):

39

"""Amulet utilities.

40

41

This class provides common utility functions that are used by Amulet

42

tests.

43

"""

44

45

def __init__(self, log_level=logging.ERROR):

46

self.log = self.get_logger(level=log_level)

47

self.ubuntu_releases = self.get_ubuntu_releases()

48

49

def get_logger(self, name="amulet-logger", level=logging.DEBUG):

50

"""Get a logger object that will log to stdout."""

51

log = logging

52

logger = log.getLogger(name)

53

fmt = log.Formatter("%(asctime)s %(funcName)s "

54

"%(levelname)s: %(message)s")

55

56

handler = log.StreamHandler(stream=sys.stdout)

57

handler.setLevel(level)

58

handler.setFormatter(fmt)

59

60

logger.addHandler(handler)

61

logger.setLevel(level)

62

63

return logger

64

65

def valid_ip(self, ip):

66

if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):

67

return True

68

else:

69

return False

70

71

def valid_url(self, url):

72

p = re.compile(

73

r'^(?:http|ftp)s?://'

74

r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa

75

r'localhost|'

76

r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'

77

r'(?::\d+)?'

78

r'(?:/?|[/?]\S+)$',

79

re.IGNORECASE)

80

if p.match(url):

81

return True

82

else:

83

return False

84

85

def get_ubuntu_release_from_sentry(self, sentry_unit):

86

"""Get Ubuntu release codename from sentry unit.

87

88

:param sentry_unit: amulet sentry/service unit pointer

89

:returns: list of strings - release codename, failure message

90

"""

91

msg = None

92

cmd = 'lsb_release -cs'

93

release, code = sentry_unit.run(cmd)

94

if code == 0:

95

self.log.debug('{} lsb_release: {}'.format(

96

sentry_unit.info['unit_name'], release))

97

else:

98

msg = ('{} `{}` returned {} '

99

'{}'.format(sentry_unit.info['unit_name'],

100

cmd, release, code))

101

if release not in self.ubuntu_releases:

102

msg = ("Release ({}) not found in Ubuntu releases "

103

"({})".format(release, self.ubuntu_releases))

104

return release, msg

105

106

def validate_services(self, commands):

107

"""Validate that lists of commands succeed on service units. Can be

108

used to verify system services are running on the corresponding

109

service units.

110

111

:param commands: dict with sentry keys and arbitrary command list vals

112

:returns: None if successful, Failure string message otherwise

113

"""

114

self.log.debug('Checking status of system services...')

115

116

# /!\ DEPRECATION WARNING (beisner):

117

# New and existing tests should be rewritten to use

118

# validate_services_by_name() as it is aware of init systems.

119

self.log.warn('DEPRECATION WARNING: use '

120

'validate_services_by_name instead of validate_services '

121

'due to init system differences.')

122

123

for k, v in six.iteritems(commands):

124

for cmd in v:

125

output, code = k.run(cmd)

126

self.log.debug('{} `{}` returned '

127

'{}'.format(k.info['unit_name'],

128

cmd, code))

129

if code != 0:

130

return "command `{}` returned {}".format(cmd, str(code))

131

return None

132

133

def validate_services_by_name(self, sentry_services):

134

"""Validate system service status by service name, automatically

135

detecting init system based on Ubuntu release codename.

136

137

:param sentry_services: dict with sentry keys and svc list values

138

:returns: None if successful, Failure string message otherwise

139

"""

140

self.log.debug('Checking status of system services...')

141

142

# Point at which systemd became a thing

143

systemd_switch = self.ubuntu_releases.index('vivid')

144

145

for sentry_unit, services_list in six.iteritems(sentry_services):

146

# Get lsb_release codename from unit

147

release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)

148

if ret:

149

return ret

150

151

for service_name in services_list:

152

if (self.ubuntu_releases.index(release) >= systemd_switch or

153

service_name in ['rabbitmq-server', 'apache2']):

154

# init is systemd (or regular sysv)

155

cmd = 'sudo service {} status'.format(service_name)

156

output, code = sentry_unit.run(cmd)

157

service_running = code == 0

158

elif self.ubuntu_releases.index(release) < systemd_switch:

159

# init is upstart

160

cmd = 'sudo status {}'.format(service_name)

161

output, code = sentry_unit.run(cmd)

162

service_running = code == 0 and "start/running" in output

163

164

self.log.debug('{} `{}` returned '

165

'{}'.format(sentry_unit.info['unit_name'],

166

cmd, code))

167

if not service_running:

168

return u"command `{}` returned {} {}".format(

169

cmd, output, str(code))

170

return None

171

172

def _get_config(self, unit, filename):

173

"""Get a ConfigParser object for parsing a unit's config file."""

174

file_contents = unit.file_contents(filename)

175

176

# NOTE(beisner): by default, ConfigParser does not handle options

177

# with no value, such as the flags used in the mysql my.cnf file.

178

# https://bugs.python.org/issue7005

179

config = configparser.ConfigParser(allow_no_value=True)

180

config.readfp(io.StringIO(file_contents))

181

return config

182

183

def validate_config_data(self, sentry_unit, config_file, section,

184

expected):

185

"""Validate config file data.

186

187

Verify that the specified section of the config file contains

188

the expected option key:value pairs.

189

190

Compare expected dictionary data vs actual dictionary data.

191

The values in the 'expected' dictionary can be strings, bools, ints,

192

longs, or can be a function that evaluates a variable and returns a

193

bool.

194

"""

195

self.log.debug('Validating config file data ({} in {} on {})'

196

'...'.format(section, config_file,

197

sentry_unit.info['unit_name']))

198

config = self._get_config(sentry_unit, config_file)

199

200

if section != 'DEFAULT' and not config.has_section(section):

201

return "section [{}] does not exist".format(section)

202

203

for k in expected.keys():

204

if not config.has_option(section, k):

205

return "section [{}] is missing option {}".format(section, k)

206

207

actual = config.get(section, k)

208

v = expected[k]

209

if (isinstance(v, six.string_types) or

210

isinstance(v, bool) or

211

isinstance(v, six.integer_types)):

212

# handle explicit values

213

if actual != v:

214

return "section [{}] {}:{} != expected {}:{}".format(

215

section, k, actual, k, expected[k])

216

# handle function pointers, such as not_null or valid_ip

217

elif not v(actual):

218

return "section [{}] {}:{} != expected {}:{}".format(

219

section, k, actual, k, expected[k])

220

return None

221

222

def _validate_dict_data(self, expected, actual):

223

"""Validate dictionary data.

224

225

Compare expected dictionary data vs actual dictionary data.

226

The values in the 'expected' dictionary can be strings, bools, ints,

227

longs, or can be a function that evaluates a variable and returns a

228

bool.

229

"""

230

self.log.debug('actual: {}'.format(repr(actual)))

231

self.log.debug('expected: {}'.format(repr(expected)))

232

233

for k, v in six.iteritems(expected):

234

if k in actual:

235

if (isinstance(v, six.string_types) or

236

isinstance(v, bool) or

237

isinstance(v, six.integer_types)):

238

# handle explicit values

239

if v != actual[k]:

240

return "{}:{}".format(k, actual[k])

241

# handle function pointers, such as not_null or valid_ip

242

elif not v(actual[k]):

243

return "{}:{}".format(k, actual[k])

244

else:

245

return "key '{}' does not exist".format(k)

246

return None

247

248

def validate_relation_data(self, sentry_unit, relation, expected):

249

"""Validate actual relation data based on expected relation data."""

250

actual = sentry_unit.relation(relation[0], relation[1])

251

return self._validate_dict_data(expected, actual)

252

253

def _validate_list_data(self, expected, actual):

254

"""Compare expected list vs actual list data."""

255

for e in expected:

256

if e not in actual:

257

return "expected item {} not found in actual list".format(e)

258

return None

259

260

def not_null(self, string):

261

if string is not None:

262

return True

263

else:

264

return False

265

266

def _get_file_mtime(self, sentry_unit, filename):

267

"""Get last modification time of file."""

268

return sentry_unit.file_stat(filename)['mtime']

269

270

def _get_dir_mtime(self, sentry_unit, directory):

271

"""Get last modification time of directory."""

272

return sentry_unit.directory_stat(directory)['mtime']

273

274

def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):

275

"""Get start time of a process based on the last modification time

276

of the /proc/pid directory.

277

278

:sentry_unit: The sentry unit to check for the service on

279

:service: service name to look for in process table

280

:pgrep_full: [Deprecated] Use full command line search mode with pgrep

281

:returns: epoch time of service process start

282

:param commands: list of bash commands

283

:param sentry_units: list of sentry unit pointers

284

:returns: None if successful; Failure message otherwise

285

"""

286

if pgrep_full is not None:

287

# /!\ DEPRECATION WARNING (beisner):

288

# No longer implemented, as pidof is now used instead of pgrep.

289

# https://bugs.launchpad.net/charm-helpers/+bug/1474030

290

self.log.warn('DEPRECATION WARNING: pgrep_full bool is no '

291

'longer implemented re: lp 1474030.')

292

293

pid_list = self.get_process_id_list(sentry_unit, service)

294

pid = pid_list[0]

295

proc_dir = '/proc/{}'.format(pid)

296

self.log.debug('Pid for {} on {}: {}'.format(

297

service, sentry_unit.info['unit_name'], pid))

298

299

return self._get_dir_mtime(sentry_unit, proc_dir)

300

301

def service_restarted(self, sentry_unit, service, filename,

302

pgrep_full=None, sleep_time=20):

303

"""Check if service was restarted.

304

305

Compare a service's start time vs a file's last modification time

306

(such as a config file for that service) to determine if the service

307

has been restarted.

308

"""

309

# /!\ DEPRECATION WARNING (beisner):

310

# This method is prone to races in that no before-time is known.

311

# Use validate_service_config_changed instead.

312

313

# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now

314

# used instead of pgrep. pgrep_full is still passed through to ensure

315

# deprecation WARNS. lp1474030

316

self.log.warn('DEPRECATION WARNING: use '

317

'validate_service_config_changed instead of '

318

'service_restarted due to known races.')

319

320

time.sleep(sleep_time)

321

if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=

322

self._get_file_mtime(sentry_unit, filename)):

323

return True

324

else:

325

return False

326

327

def service_restarted_since(self, sentry_unit, mtime, service,

328

pgrep_full=None, sleep_time=20,

329

retry_count=30, retry_sleep_time=10):

330

"""Check if service was been started after a given time.

331

332

Args:

333

sentry_unit (sentry): The sentry unit to check for the service on

334

mtime (float): The epoch time to check against

335

service (string): service name to look for in process table

336

pgrep_full: [Deprecated] Use full command line search mode with pgrep

337

sleep_time (int): Initial sleep time (s) before looking for file

338

retry_sleep_time (int): Time (s) to sleep between retries

339

retry_count (int): If file is not found, how many times to retry

340

341

Returns:

342

bool: True if service found and its start time it newer than mtime,

343

False if service is older than mtime or if service was

344

not found.

345

"""

346

# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now

347

# used instead of pgrep. pgrep_full is still passed through to ensure

348

# deprecation WARNS. lp1474030

349

350

unit_name = sentry_unit.info['unit_name']

351

self.log.debug('Checking that %s service restarted since %s on '

352

'%s' % (service, mtime, unit_name))

353

time.sleep(sleep_time)

354

proc_start_time = None

355

tries = 0

356

while tries <= retry_count and not proc_start_time:

357

try:

358

proc_start_time = self._get_proc_start_time(sentry_unit,

359

service,

360

pgrep_full)

361

self.log.debug('Attempt {} to get {} proc start time on {} '

362

'OK'.format(tries, service, unit_name))

363

except IOError as e:

364

# NOTE(beisner) - race avoidance, proc may not exist yet.

365

# https://bugs.launchpad.net/charm-helpers/+bug/1474030

366

self.log.debug('Attempt {} to get {} proc start time on {} '

367

'failed\n{}'.format(tries, service,

368

unit_name, e))

369

time.sleep(retry_sleep_time)

370

tries += 1

371

372

if not proc_start_time:

373

self.log.warn('No proc start time found, assuming service did '

374

'not start')

375

return False

376

if proc_start_time >= mtime:

377

self.log.debug('Proc start time is newer than provided mtime'

378

'(%s >= %s) on %s (OK)' % (proc_start_time,

379

mtime, unit_name))

380

return True

381

else:

382

self.log.warn('Proc start time (%s) is older than provided mtime '

383

'(%s) on %s, service did not '

384

'restart' % (proc_start_time, mtime, unit_name))

385

return False

386

387

def config_updated_since(self, sentry_unit, filename, mtime,

388

sleep_time=20, retry_count=30,

389

retry_sleep_time=10):

390

"""Check if file was modified after a given time.

391

392

Args:

393

sentry_unit (sentry): The sentry unit to check the file mtime on

394

filename (string): The file to check mtime of

395

mtime (float): The epoch time to check against

396

sleep_time (int): Initial sleep time (s) before looking for file

397

retry_sleep_time (int): Time (s) to sleep between retries

398

retry_count (int): If file is not found, how many times to retry

399

400

Returns:

401

bool: True if file was modified more recently than mtime, False if

402

file was modified before mtime, or if file not found.

403

"""

404

unit_name = sentry_unit.info['unit_name']

405

self.log.debug('Checking that %s updated since %s on '

406

'%s' % (filename, mtime, unit_name))

407

time.sleep(sleep_time)

408

file_mtime = None

409

tries = 0

410

while tries <= retry_count and not file_mtime:

411

try:

412

file_mtime = self._get_file_mtime(sentry_unit, filename)

413

self.log.debug('Attempt {} to get {} file mtime on {} '

414

'OK'.format(tries, filename, unit_name))

415

except IOError as e:

416

# NOTE(beisner) - race avoidance, file may not exist yet.

417

# https://bugs.launchpad.net/charm-helpers/+bug/1474030

418

self.log.debug('Attempt {} to get {} file mtime on {} '

419

'failed\n{}'.format(tries, filename,

420

unit_name, e))

421

time.sleep(retry_sleep_time)

422

tries += 1

423

424

if not file_mtime:

425

self.log.warn('Could not determine file mtime, assuming '

426

'file does not exist')

427

return False

428

429

if file_mtime >= mtime:

430

self.log.debug('File mtime is newer than provided mtime '

431

'(%s >= %s) on %s (OK)' % (file_mtime,

432

mtime, unit_name))

433

return True

434

else:

435

self.log.warn('File mtime is older than provided mtime'

436

'(%s < on %s) on %s' % (file_mtime,

437

mtime, unit_name))

438

return False

439

440

def validate_service_config_changed(self, sentry_unit, mtime, service,

441

filename, pgrep_full=None,

442

sleep_time=20, retry_count=30,

443

retry_sleep_time=10):

444

"""Check service and file were updated after mtime

445

446

Args:

447

sentry_unit (sentry): The sentry unit to check for the service on

448

mtime (float): The epoch time to check against

449

service (string): service name to look for in process table

450

filename (string): The file to check mtime of

451

pgrep_full: [Deprecated] Use full command line search mode with pgrep

452

sleep_time (int): Initial sleep in seconds to pass to test helpers

453

retry_count (int): If service is not found, how many times to retry

454

retry_sleep_time (int): Time in seconds to wait between retries

455

456

Typical Usage:

457

u = OpenStackAmuletUtils(ERROR)

458

...

459

mtime = u.get_sentry_time(self.cinder_sentry)

460

self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})

461

if not u.validate_service_config_changed(self.cinder_sentry,

462

mtime,

463

'cinder-api',

464

'/etc/cinder/cinder.conf')

465

amulet.raise_status(amulet.FAIL, msg='update failed')

466

Returns:

467

bool: True if both service and file where updated/restarted after

468

mtime, False if service is older than mtime or if service was

469

not found or if filename was modified before mtime.

470

"""

471

472

# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now

473

# used instead of pgrep. pgrep_full is still passed through to ensure

474

# deprecation WARNS. lp1474030

475

476

service_restart = self.service_restarted_since(

477

sentry_unit, mtime,

478

service,

479

pgrep_full=pgrep_full,

480

sleep_time=sleep_time,

481

retry_count=retry_count,

482

retry_sleep_time=retry_sleep_time)

483

484

config_update = self.config_updated_since(

485

sentry_unit,

486

filename,

487

mtime,

488

sleep_time=sleep_time,

489

retry_count=retry_count,

490

retry_sleep_time=retry_sleep_time)

491

492

return service_restart and config_update

493

494

def get_sentry_time(self, sentry_unit):

495

"""Return current epoch time on a sentry"""

496

cmd = "date +'%s'"

497

return float(sentry_unit.run(cmd)[0])

498

499

def relation_error(self, name, data):

500

return 'unexpected relation data in {} - {}'.format(name, data)

501

502

def endpoint_error(self, name, data):

503

return 'unexpected endpoint data in {} - {}'.format(name, data)

504

505

def get_ubuntu_releases(self):

506

"""Return a list of all Ubuntu releases in order of release."""

507

_d = distro_info.UbuntuDistroInfo()

508

_release_list = _d.all

509

return _release_list

510

511

def file_to_url(self, file_rel_path):

512

"""Convert a relative file path to a file URL."""

513

_abs_path = os.path.abspath(file_rel_path)

514

return urlparse.urlparse(_abs_path, scheme='file').geturl()

515

516

def check_commands_on_units(self, commands, sentry_units):

517

"""Check that all commands in a list exit zero on all

518

sentry units in a list.

519

520

:param commands: list of bash commands

521

:param sentry_units: list of sentry unit pointers

522

:returns: None if successful; Failure message otherwise

523

"""

524

self.log.debug('Checking exit codes for {} commands on {} '

525

'sentry units...'.format(len(commands),

526

len(sentry_units)))

527

for sentry_unit in sentry_units:

528

for cmd in commands:

529

output, code = sentry_unit.run(cmd)

530

if code == 0:

531

self.log.debug('{} `{}` returned {} '

532

'(OK)'.format(sentry_unit.info['unit_name'],

533

cmd, code))

534

else:

535

return ('{} `{}` returned {} '

536

'{}'.format(sentry_unit.info['unit_name'],

537

cmd, code, output))

538

return None

539

540

def get_process_id_list(self, sentry_unit, process_name,

541

expect_success=True):

542

"""Get a list of process ID(s) from a single sentry juju unit

543

for a single process name.

544

545

:param sentry_unit: Amulet sentry instance (juju unit)

546

:param process_name: Process name

547

:param expect_success: If False, expect the PID to be missing,

548

raise if it is present.

549

:returns: List of process IDs

550

"""

551

cmd = 'pidof -x {}'.format(process_name)

552

if not expect_success:

553

cmd += " || exit 0 && exit 1"

554

output, code = sentry_unit.run(cmd)

555

if code != 0:

556

msg = ('{} `{}` returned {} '

557

'{}'.format(sentry_unit.info['unit_name'],

558

cmd, code, output))

559

amulet.raise_status(amulet.FAIL, msg=msg)

560

return str(output).split()

561

562

def get_unit_process_ids(self, unit_processes, expect_success=True):

563

"""Construct a dict containing unit sentries, process names, and

564

process IDs.

565

566

:param unit_processes: A dictionary of Amulet sentry instance

567

to list of process names.

568

:param expect_success: if False expect the processes to not be

569

running, raise if they are.

570

:returns: Dictionary of Amulet sentry instance to dictionary

571

of process names to PIDs.

572

"""

573

pid_dict = {}

574

for sentry_unit, process_list in six.iteritems(unit_processes):

575

pid_dict[sentry_unit] = {}

576

for process in process_list:

577

pids = self.get_process_id_list(

578

sentry_unit, process, expect_success=expect_success)

579

pid_dict[sentry_unit].update({process: pids})

580

return pid_dict

581

582

def validate_unit_process_ids(self, expected, actual):

583

"""Validate process id quantities for services on units."""

584

self.log.debug('Checking units for running processes...')

585

self.log.debug('Expected PIDs: {}'.format(expected))

586

self.log.debug('Actual PIDs: {}'.format(actual))

587

588

if len(actual) != len(expected):

589

return ('Unit count mismatch. expected, actual: {}, '

590

'{} '.format(len(expected), len(actual)))

591

592

for (e_sentry, e_proc_names) in six.iteritems(expected):

593

e_sentry_name = e_sentry.info['unit_name']

594

if e_sentry in actual.keys():

595

a_proc_names = actual[e_sentry]

596

else:

597

return ('Expected sentry ({}) not found in actual dict data.'

598

'{}'.format(e_sentry_name, e_sentry))

599

600

if len(e_proc_names.keys()) != len(a_proc_names.keys()):

601

return ('Process name count mismatch. expected, actual: {}, '

602

'{}'.format(len(expected), len(actual)))

603

604

for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \

605

zip(e_proc_names.items(), a_proc_names.items()):

606

if e_proc_name != a_proc_name:

607

return ('Process name mismatch. expected, actual: {}, '

608

'{}'.format(e_proc_name, a_proc_name))

609

610

a_pids_length = len(a_pids)

611

fail_msg = ('PID count mismatch. {} ({}) expected, actual: '

612

'{}, {} ({})'.format(e_sentry_name, e_proc_name,

613

e_pids_length, a_pids_length,

614

a_pids))

615

616

# If expected is not bool, ensure PID quantities match

617

if not isinstance(e_pids_length, bool) and \

618

a_pids_length != e_pids_length:

619

return fail_msg

620

# If expected is bool True, ensure 1 or more PIDs exist

621

elif isinstance(e_pids_length, bool) and \

622

e_pids_length is True and a_pids_length < 1:

623

return fail_msg

624

# If expected is bool False, ensure 0 PIDs exist

625

elif isinstance(e_pids_length, bool) and \

626

e_pids_length is False and a_pids_length != 0:

627

return fail_msg

628

else:

629

self.log.debug('PID check OK: {} {} {}: '

630

'{}'.format(e_sentry_name, e_proc_name,

631

e_pids_length, a_pids))

632

return None

633

634

def validate_list_of_identical_dicts(self, list_of_dicts):

635

"""Check that all dicts within a list are identical."""

636

hashes = []

637

for _dict in list_of_dicts:

638

hashes.append(hash(frozenset(_dict.items())))

639

640

self.log.debug('Hashes: {}'.format(hashes))

641

if len(set(hashes)) == 1:

642

self.log.debug('Dicts within list are identical')

643

else:

644

return 'Dicts within list are not identical'

645

646

return None

647

648

def validate_sectionless_conf(self, file_contents, expected):

649

"""A crude conf parser. Useful to inspect configuration files which

650

do not have section headers (as would be necessary in order to use

651

the configparser). Such as openstack-dashboard or rabbitmq confs."""

652

for line in file_contents.split('\n'):

653

if '=' in line:

654

args = line.split('=')

655

if len(args) <= 1:

656

continue

657

key = args[0].strip()

658

value = args[1].strip()

659

if key in expected.keys():

660

if expected[key] != value:

661

msg = ('Config mismatch. Expected, actual: {}, '

662

'{}'.format(expected[key], value))

663

amulet.raise_status(amulet.FAIL, msg=msg)

664

665

def get_unit_hostnames(self, units):

666

"""Return a dict of juju unit names to hostnames."""

667

host_names = {}

668

for unit in units:

669

host_names[unit.info['unit_name']] = \

670

str(unit.file_contents('/etc/hostname').strip())

671

self.log.debug('Unit host names: {}'.format(host_names))

672

return host_names

673

674

def run_cmd_unit(self, sentry_unit, cmd):

675

"""Run a command on a unit, return the output and exit code."""

676

output, code = sentry_unit.run(cmd)

677

if code == 0:

678

self.log.debug('{} `{}` command returned {} '

679

'(OK)'.format(sentry_unit.info['unit_name'],

680

cmd, code))

681

else:

682

msg = ('{} `{}` command returned {} '

683

'{}'.format(sentry_unit.info['unit_name'],

684

cmd, code, output))

685

amulet.raise_status(amulet.FAIL, msg=msg)

686

return str(output), code

687

688

def file_exists_on_unit(self, sentry_unit, file_name):

689

"""Check if a file exists on a unit."""

690

try:

691

sentry_unit.file_stat(file_name)

692

return True

693

except IOError:

694

return False

695

except Exception as e:

696

msg = 'Error checking file {}: {}'.format(file_name, e)

697

amulet.raise_status(amulet.FAIL, msg=msg)

698

699

def file_contents_safe(self, sentry_unit, file_name,

700

max_wait=60, fatal=False):

701

"""Get file contents from a sentry unit. Wrap amulet file_contents

702

with retry logic to address races where a file checks as existing,

703

but no longer exists by the time file_contents is called.

704

Return None if file not found. Optionally raise if fatal is True."""

705

unit_name = sentry_unit.info['unit_name']

706

file_contents = False

707

tries = 0

708

while not file_contents and tries < (max_wait / 4):

709

try:

710

file_contents = sentry_unit.file_contents(file_name)

711

except IOError:

712

self.log.debug('Attempt {} to open file {} from {} '

713

'failed'.format(tries, file_name,

714

unit_name))

715

time.sleep(4)

716

tries += 1

717

718

if file_contents:

719

return file_contents

720

elif not fatal:

721

return None

722

elif fatal:

723

msg = 'Failed to get file contents from unit.'

724

amulet.raise_status(amulet.FAIL, msg)

725

726

def port_knock_tcp(self, host="localhost", port=22, timeout=15):

727

"""Open a TCP socket to check for a listening sevice on a host.

728

729

:param host: host name or IP address, default to localhost

730

:param port: TCP port number, default to 22

731

:param timeout: Connect timeout, default to 15 seconds

732

:returns: True if successful, False if connect failed

733

"""

734

735

# Resolve host name if possible

736

try:

737

connect_host = socket.gethostbyname(host)

738

host_human = "{} ({})".format(connect_host, host)

739

except socket.error as e:

740

self.log.warn('Unable to resolve address: '

741

'{} ({}) Trying anyway!'.format(host, e))

742

connect_host = host

743

host_human = connect_host

744

745

# Attempt socket connection

746

try:

747

knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

748

knock.settimeout(timeout)

749

knock.connect((connect_host, port))

750

knock.close()

751

self.log.debug('Socket connect OK for host '

752

'{} on port {}.'.format(host_human, port))

753

return True

754

except socket.error as e:

755

self.log.debug('Socket connect FAIL for'

756

' {} port {} ({})'.format(host_human, port, e))

757

return False

758

759

def port_knock_units(self, sentry_units, port=22,

760

timeout=15, expect_success=True):

761

"""Open a TCP socket to check for a listening sevice on each

762

listed juju unit.

763

764

:param sentry_units: list of sentry unit pointers

765

:param port: TCP port number, default to 22

766

:param timeout: Connect timeout, default to 15 seconds

767

:expect_success: True by default, set False to invert logic

768

:returns: None if successful, Failure message otherwise

769

"""

770

for unit in sentry_units:

771

host = unit.info['public-address']

772

connected = self.port_knock_tcp(host, port, timeout)

773

if not connected and expect_success:

774

return 'Socket connect failed.'

775

elif connected and not expect_success:

776

return 'Socket connected unexpectedly.'

777

778

def get_uuid_epoch_stamp(self):

779

"""Returns a stamp string based on uuid4 and epoch time. Useful in

780

generating test messages which need to be unique-ish."""

781

return '[{}-{}]'.format(uuid.uuid4(), time.time())

782

783

# amulet juju action helpers:

784

def run_action(self, unit_sentry, action,

785

_check_output=subprocess.check_output):

786

"""Run the named action on a given unit sentry.

787

788

_check_output parameter is used for dependency injection.

789

790

@return action_id.

791

"""

792

unit_id = unit_sentry.info["unit_name"]

793

command = ["juju", "action", "do", "--format=json", unit_id, action]

794

self.log.info("Running command: %s\n" % " ".join(command))

795

output = _check_output(command, universal_newlines=True)

796

data = json.loads(output)

797

action_id = data[u'Action queued with id']

798

return action_id

799

800

def wait_on_action(self, action_id, _check_output=subprocess.check_output):

801

"""Wait for a given action, returning if it completed or not.

802

803

_check_output parameter is used for dependency injection.

804

"""

805

command = ["juju", "action", "fetch", "--format=json", "--wait=0",

806

action_id]

807

output = _check_output(command, universal_newlines=True)

808

data = json.loads(output)

809

return data.get(u"status") == "completed"

810

811

def status_get(self, unit):

812

"""Return the current service status of this unit."""

813

raw_status, return_code = unit.run(

814

"status-get --format=json --include-data")

815

if return_code != 0:

816

return ("unknown", "")

817

status = json.loads(raw_status)

818

return (status["status"], status["message"])