~nottrobin/charms/precise/block-storage-broker/ensure-python-apt : revision 48

1

"""Common python utilities for the ec2 provider"""

2

3

from charmhelpers.core import hookenv

4

import subprocess

5

import os

6

import sys

7

from time import sleep

8

9

ENVIRONMENT_MAP = {

10

"ec2": {"endpoint": "EC2_URL", "key": "EC2_ACCESS_KEY",

11

"secret": "EC2_SECRET_KEY"},

12

"nova": {"endpoint": "OS_AUTH_URL", "region": "OS_REGION_NAME",

13

"tenant": "OS_TENANT_NAME", "key": "OS_USERNAME",

14

"secret": "OS_PASSWORD"}}

15

16

REQUIRED_CONFIG_OPTIONS = {

17

"ec2": ["endpoint", "key", "secret"],

18

"nova": ["endpoint", "region", "tenant", "key", "secret"]}

19

20

PROVIDER_COMMANDS = {

21

"ec2": {"validate": "euca-describe-instances",

22

"detach": "euca-detach-volume -i %s %s"},

23

"nova": {"validate": "nova list",

24

"detach": "nova volume-detach %s %s"}}

25

26

27

class StorageServiceUtil(object):

28

"""Interact with an underlying cloud storage provider.

29

Create, attach, label and detach storage volumes using EC2 or nova APIs.

30

"""

31

provider = None

32

environment_map = None

33

required_config_options = None

34

commands = None

35

36

def __init__(self, provider):

37

self.provider = provider

38

if provider not in ENVIRONMENT_MAP:

39

hookenv.log(

40

"ERROR: Invalid charm configuration setting for provider. "

41

"'%s' must be one of: %s" %

42

(provider, ", ".join(ENVIRONMENT_MAP.keys())),

43

hookenv.ERROR)

44

sys.exit(1)

45

self.environment_map = ENVIRONMENT_MAP[provider]

46

self.commands = PROVIDER_COMMANDS[provider]

47

self.required_config_options = REQUIRED_CONFIG_OPTIONS[provider]

48

if provider == "ec2":

49

import euca2ools.commands.euca.describevolumes as getvolumes

50

import euca2ools.commands.euca.describeinstances as getinstances

51

self.ec2_volume_class = getvolumes.DescribeVolumes

52

self.ec2_instance_class = getinstances.DescribeInstances

53

54

def load_environment(self):

55

"""

56

Source our credentials from the configuration definitions into our

57

operating environment

58

"""

59

config_data = hookenv.config()

60

for option in self.required_config_options:

61

environment_variable = self.environment_map[option]

62

os.environ[environment_variable] = config_data[option].strip()

63

self.validate_credentials()

64

65

def validate_credentials(self):

66

"""

67

Attempt to contact the respective ec2 or nova volume service or exit(1)

68

"""

69

try:

70

subprocess.check_call(self.commands["validate"], shell=True)

71

except subprocess.CalledProcessError, e:

72

hookenv.log(

73

"ERROR: Charm configured credentials can't access endpoint. "

74

"%s" % str(e),

75

hookenv.ERROR)

76

sys.exit(1)

77

hookenv.log(

78

"Validated charm configuration credentials have access to block "

79

"storage service")

80

81

def describe_volumes(self, volume_id=None):

82

method = getattr(self, "_%s_describe_volumes" % self.provider)

83

return method(volume_id)

84

85

def get_volume_id(self, volume_designation=None):

86

"""Return the ec2 or nova volume id associated with this unit

87

88

Optionally, C{volume_designation} can be either a volume-id or

89

volume-display-name and the matching C{volume-id} will be returned.

90

If no matching volume is return C{None}.

91

"""

92

matches = []

93

volumes = self.describe_volumes()

94

if volume_designation:

95

token = volume_designation

96

else:

97

# Try to find volume label containing remote_unit name

98

token = hookenv.remote_unit()

99

for volume_id in volumes.keys():

100

volume = volumes[volume_id]

101

# Get volume by name or volume-id

102

volume_name = volume["tags"].get("volume_name", "")

103

if token == volume_id:

104

matches.append(volume_id)

105

elif token in volume_name:

106

matches.append(volume_id)

107

if len(matches) > 1:

108

hookenv.log(

109

"Error: Multiple volumes are associated with "

110

"%s. Cannot get_volume_id." % token, hookenv.ERROR)

111

sys.exit(1)

112

elif matches:

113

return matches[0]

114

return None

115

116

def attach_volume(self, instance_id, volume_id=None, size=None,

117

volume_label=None):

118

"""

119

Create and attach a volume to the remote unit if none exists.

120

121

Attempt to attach and validate the attached volume 10

122

times. If unable to resolve the attach issues, exit in error and log

123

the error.

124

Log errors if the volume is in an unsupported state, and if C{in-use}

125

report it is already attached.

126

127

Return the device-path of the attached volume to the caller.

128

"""

129

self.load_environment() # Will fail if invalid environment

130

remote_unit = hookenv.remote_unit()

131

if volume_label is None:

132

volume_label = generate_volume_label(remote_unit)

133

if volume_id:

134

volume = self.describe_volumes(volume_id)

135

if not volume:

136

hookenv.log(

137

"Requested volume-id (%s) does not exist. Unable to "

138

"associate storage with %s" % (volume_id, remote_unit),

139

hookenv.ERROR)

140

sys.exit(1)

141

142

# Validate that current volume status is supported

143

while volume["status"] == "attaching":

144

hookenv.log("Volume %s still attaching. Waiting." % volume_id)

145

sleep(5)

146

volume = self.describe_volumes(volume_id)

147

148

if volume["status"] == "in-use":

149

hookenv.log("Volume %s already attached. Done" % volume_id)

150

return volume["device"] # The device path on the instance

151

if volume["status"] != "available":

152

hookenv.log(

153

"Cannot attach volume. Volume has unsupported status: "

154

"%s" % volume["status"], hookenv.ERROR)

155

sys.exit(1)

156

else:

157

# No volume_id, create a new volume if one isn't already created

158

# for the principal of this JUJU_REMOTE_UNIT

159

volume_id = self.get_volume_id(volume_label)

160

if not volume_id:

161

create = getattr(self, "_%s_create_volume" % self.provider)

162

if not size:

163

size = hookenv.config("default_volume_size")

164

volume_id = create(size, volume_label, instance_id)

165

166

device = None

167

hookenv.log("Attaching %s (%s)" % (volume_label, volume_id))

168

for x in range(10):

169

volume = self.describe_volumes(volume_id)

170

if volume["status"] == "in-use":

171

return volume["device"] # The device path on the instance

172

if volume["status"] == "available":

173

attach = getattr(self, "_%s_attach_volume" % self.provider)

174

device = attach(instance_id, volume_id)

175

break

176

else:

177

sleep(5)

178

if not device:

179

hookenv.log(

180

"ERROR: Unable to discover device attached by "

181

"euca-attach-volume",

182

hookenv.ERROR)

183

sys.exit(1)

184

return device

185

186

def detach_volume(self, volume_label):

187

"""Detach a volume from remote unit if present"""

188

self.load_environment() # Will fail if invalid environment

189

volume_id = self.get_volume_id(volume_label)

190

191

if volume_id:

192

volume = self.describe_volumes(volume_id)

193

else:

194

hookenv.log("Cannot find volume name to detach, done")

195

return

196

197

if volume["status"] == "available":

198

hookenv.log("Volume (%s) already detached. Done" % volume_id)

199

return

200

201

hookenv.log(

202

"Detaching volume (%s) from instance %s" %

203

(volume_id, volume["instance_id"]))

204

try:

205

subprocess.check_call(

206

self.commands["detach"] % (volume["instance_id"], volume_id),

207

shell=True)

208

except subprocess.CalledProcessError, e:

209

hookenv.log(

210

"ERROR: Couldn't detach volume. %s" % str(e), hookenv.ERROR)

211

sys.exit(1)

212

return

213

214

# EC2-specific methods

215

def _ec2_create_tag(self, volume_id, tag_name, tag_value=None):

216

"""Attach a tag and optional C{tag_value} to the given C{volume_id}"""

217

tag_string = tag_name

218

if tag_value:

219

tag_string += "=%s" % tag_value

220

command = 'euca-create-tags %s --tag "%s"' % (volume_id, tag_string)

221

222

try:

223

subprocess.check_call(command, shell=True)

224

except subprocess.CalledProcessError, e:

225

hookenv.log(

226

"ERROR: Couldn't add tags to the resource. %s" % str(e),

227

hookenv.ERROR)

228

sys.exit(1)

229

hookenv.log("Tagged (%s) to %s." % (tag_string, volume_id))

230

231

def _ec2_describe_instances(self, instance_id=None):

232

"""

233

Use euca2ools libraries to describe instances and return a C{dict}

234

"""

235

result = {}

236

try:

237

command = self.ec2_instance_class()

238

reservations = command.main()

239

except SystemExit:

240

hookenv.log(

241

"ERROR: Couldn't contact EC2 using euca-describe-instances",

242

hookenv.ERROR)

243

sys.exit(1)

244

for reservation in reservations:

245

for inst in reservation.instances:

246

result[inst.id] = {

247

"ip-address": inst.ip_address, "image-id": inst.image_id,

248

"instance-type": inst.image_id, "kernel": inst.kernel,

249

"private-dns-name": inst.private_dns_name,

250

"public-dns-name": inst.public_dns_name,

251

"reservation-id": reservation.id,

252

"state": inst.state, "tags": inst.tags,

253

"availability_zone": inst.placement}

254

if instance_id:

255

if instance_id in result:

256

return result[instance_id]

257

return {}

258

return result

259

260

def _ec2_describe_volumes(self, volume_id=None):

261

"""

262

Use euca2ools libraries to describe volumes and return a C{dict}

263

"""

264

result = {}

265

try:

266

command = self.ec2_volume_class()

267

volumes = command.main()

268

except SystemExit:

269

hookenv.log(

270

"ERROR: Couldn't contact EC2 using euca-describe-volumes",

271

hookenv.ERROR)

272

sys.exit(1)

273

for volume in volumes:

274

result[volume.id] = {

275

"device": "",

276

"instance_id": "",

277

"size": volume.size,

278

"snapshot_id": volume.snapshot_id,

279

"status": volume.status,

280

"tags": volume.tags,

281

"id": volume.id,

282

"availability_zone": volume.zone}

283

if "volume_name" in volume.tags:

284

result[volume.id]["volume_label"] = volume.tags["volume_name"]

285

else:

286

result[volume.id]["tags"]["volume_name"] = ""

287

result[volume.id]["volume_label"] = ""

288

if volume.status == "in-use":

289

result[volume.id]["instance_id"] = (

290

volume.attach_data.instance_id)

291

result[volume.id]["device"] = volume.attach_data.device

292

if volume_id:

293

if volume_id in result:

294

return result[volume_id]

295

return {}

296

return result

297

298

def _ec2_create_volume(self, size, volume_label, instance_id):

299

"""Create an EC2 volume with a specific C{size} and C{volume_label}"""

300

# Volumes need to be in the same zone as the instance

301

hookenv.log(

302

"Creating a %sGig volume named (%s) for instance %s" %

303

(size, volume_label, instance_id))

304

instance = self._ec2_describe_instances(instance_id)

305

if not instance:

306

config_data = hookenv.config()

307

hookenv.log(

308

"ERROR: Could not create volume for instance %s. No instance "

309

"details discovered by euca-describe-instances. Maybe the "

310

"charm configured endpoint %s is not valid for this region." %

311

(instance_id, config_data["endpoint"]), hookenv.ERROR)

312

sys.exit(1)

313

314

try:

315

output = subprocess.check_output(

316

"euca-create-volume -z %s -s %s" %

317

(instance["availability_zone"], size), shell=True)

318

except subprocess.CalledProcessError, e:

319

hookenv.log("ERROR: %s" % str(e), hookenv.ERROR)

320

sys.exit(1)

321

322

response_type, volume_id = output.split()[:2]

323

if response_type != "VOLUME":

324

hookenv.log(

325

"ERROR: Didn't get VOLUME response from euca-create-volume. "

326

"Response: %s" % output, hookenv.ERROR)

327

sys.exit(1)

328

volume = self.describe_volumes(volume_id.strip())

329

if not volume:

330

hookenv.log(

331

"ERROR: Unable to find volume '%s'" % volume_id.strip(),

332

hookenv.ERROR)

333

sys.exit(1)

334

volume_id = volume["id"]

335

self._ec2_create_tag(volume_id, "volume_name", volume_label)

336

return volume_id

337

338

def _ec2_attach_volume(self, instance_id, volume_id):

339

"""

340

Attach an EC2 C{volume_id} to the provided C{instance_id} and return

341

the device path.

342

"""

343

device = "/dev/xvdc"

344

try:

345

subprocess.check_call(

346

"euca-attach-volume -i %s -d %s %s" %

347

(instance_id, device, volume_id), shell=True)

348

except subprocess.CalledProcessError, e:

349

hookenv.log("ERROR: %s" % str(e), hookenv.ERROR)

350

sys.exit(1)

351

return device

352

353

# Nova-specific methods

354

def _nova_volume_show(self, volume_id):

355

"""

356

Read detailed information about a C{volume_id} from nova-volume-show

357

and return a C{dict} of data we are interested in.

358

"""

359

from ast import literal_eval

360

result = {"tags": {}, "instance_id": "", "device": ""}

361

command = "nova volume-show '%s'" % volume_id

362

try:

363

output = subprocess.check_output(command, shell=True)

364

except subprocess.CalledProcessError, e:

365

hookenv.log(

366

"ERROR: Failed to get nova volume info. %s" % str(e),

367

hookenv.ERROR)

368

sys.exit(1)

369

for line in output.split("\n"):

370

if not line.strip(): # Skip empty lines

371

continue

372

if "+----" in line or "Property" in line:

373

continue

374

(_, key, value, _) = line.split("|")

375

key = key.strip()

376

value = value.strip()

377

if key in ["availability_zone", "size", "id", "snapshot_id",

378

"status"]:

379

result[key] = value

380

if key == "display_name": # added for compatibility with ec2

381

result["volume_label"] = value

382

result["tags"]["volume_label"] = value

383

if key == "attachments":

384

attachments = literal_eval(value)

385

if attachments:

386

for key, value in attachments[0].items():

387

if key in ["device"]:

388

result[key] = value

389

if key == "server_id":

390

result["instance_id"] = value

391

return result

392

393

def _nova_describe_volumes(self, volume_id=None):

394

"""Create a C{dict} describing all nova volumes"""

395

result = {}

396

command = "nova volume-list"

397

try:

398

output = subprocess.check_output(command, shell=True)

399

except subprocess.CalledProcessError, e:

400

hookenv.log("ERROR: %s" % str(e), hookenv.ERROR)

401

sys.exit(1)

402

for line in output.split("\n"):

403

if not line.strip(): # Skip empty lines

404

continue

405

if "+----" in line or "ID" in line:

406

continue

407

values = line.split("|")

408

if volume_id and values[1].strip() != volume_id:

409

continue

410

volume = values[1].strip()

411

volume_label = values[3].strip()

412

if volume_label == "None":

413

volume_label = ""

414

instance_id = values[6].strip()

415

if instance_id == "None":

416

instance_id = ""

417

result[volume] = {

418

"id": volume,

419

"tags": {"volume_name": volume_label},

420

"status": values[2].strip(),

421

"volume_label": volume_label,

422

"size": values[4].strip(),

423

"instance_id": instance_id}

424

if instance_id:

425

result[volume].update(self._nova_volume_show(volume))

426

if volume_id:

427

if volume_id in result:

428

return result[volume_id]

429

return {}

430

return result

431

432

def _nova_attach_volume(self, instance_id, volume_id):

433

"""

434

Attach a Nova C{volume_id} to the provided C{instance_id} and return

435

the device path.

436

"""

437

try:

438

device = subprocess.check_output(

439

"nova volume-attach %s %s auto | egrep -o \"/dev/vd[b-z]\"" %

440

(instance_id, volume_id), shell=True)

441

except subprocess.CalledProcessError, e:

442

hookenv.log("ERROR: %s" % str(e), hookenv.ERROR)

443

sys.exit(1)

444

if device.strip():

445

return device.strip()

446

return ""

447

448

def _nova_create_volume(self, size, volume_label, instance_id):

449

"""Create an Nova volume with a specific C{size} and C{volume_label}"""

450

hookenv.log(

451

"Creating a %sGig volume named (%s) for instance %s" %

452

(size, volume_label, instance_id))

453

try:

454

subprocess.check_call(

455

"nova volume-create --display-name '%s' %s" %

456

(volume_label, size), shell=True)

457

except subprocess.CalledProcessError, e:

458

hookenv.log("ERROR: %s" % str(e), hookenv.ERROR)

459

sys.exit(1)

460

461

volume_id = self.get_volume_id(volume_label)

462

if not volume_id:

463

hookenv.log(

464

"ERROR: Couldn't find newly created nova volume '%s'." %

465

volume_label, hookenv.ERROR)

466

sys.exit(1)

467

return volume_id

468

469

470

def generate_volume_label(remote_unit):

471

"""Create a volume label for the requesting remote unit"""

472

return "%s unit volume" % remote_unit