1
# Copyright 2014-2015 Canonical Limited.
3
# This file is part of charm-helpers.
5
# charm-helpers is free software: you can redistribute it and/or modify
6
# it under the terms of the GNU Lesser General Public License version 3 as
7
# published by the Free Software Foundation.
9
# charm-helpers is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU Lesser General Public License for more details.
14
# You should have received a copy of the GNU Lesser General Public License
15
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
25
import cinderclient.v1.client as cinder_client
26
import glanceclient.v1.client as glance_client
27
import heatclient.v1.client as heat_client
28
import keystoneclient.v2_0 as keystone_client
29
import novaclient.v1_1.client as nova_client
32
from charmhelpers.contrib.amulet.utils import (
40
class OpenStackAmuletUtils(AmuletUtils):
41
"""OpenStack amulet utilities.
43
This class inherits from AmuletUtils and has additional support
44
that is specifically for use by OpenStack charm tests.
47
def __init__(self, log_level=ERROR):
48
"""Initialize the deployment environment."""
49
super(OpenStackAmuletUtils, self).__init__(log_level)
51
def validate_endpoint_data(self, endpoints, admin_port, internal_port,
52
public_port, expected):
53
"""Validate endpoint data.
55
Validate actual endpoint data vs expected endpoint data. The ports
56
are used to find the matching endpoint.
58
self.log.debug('Validating endpoint data...')
59
self.log.debug('actual: {}'.format(repr(endpoints)))
62
self.log.debug('endpoint: {}'.format(repr(ep)))
63
if (admin_port in ep.adminurl and
64
internal_port in ep.internalurl and
65
public_port in ep.publicurl):
67
actual = {'id': ep.id,
69
'adminurl': ep.adminurl,
70
'internalurl': ep.internalurl,
71
'publicurl': ep.publicurl,
72
'service_id': ep.service_id}
73
ret = self._validate_dict_data(expected, actual)
75
return 'unexpected endpoint data - {}'.format(ret)
78
return 'endpoint not found'
80
def validate_svc_catalog_endpoint_data(self, expected, actual):
81
"""Validate service catalog endpoint data.
83
Validate a list of actual service catalog endpoints vs a list of
84
expected service catalog endpoints.
86
self.log.debug('Validating service catalog endpoint data...')
87
self.log.debug('actual: {}'.format(repr(actual)))
88
for k, v in six.iteritems(expected):
90
ret = self._validate_dict_data(expected[k][0], actual[k][0])
92
return self.endpoint_error(k, ret)
94
return "endpoint {} does not exist".format(k)
97
def validate_tenant_data(self, expected, actual):
98
"""Validate tenant data.
100
Validate a list of actual tenant data vs list of expected tenant
103
self.log.debug('Validating tenant data...')
104
self.log.debug('actual: {}'.format(repr(actual)))
108
a = {'enabled': act.enabled, 'description': act.description,
109
'name': act.name, 'id': act.id}
110
if e['name'] == a['name']:
112
ret = self._validate_dict_data(e, a)
114
return "unexpected tenant data - {}".format(ret)
116
return "tenant {} does not exist".format(e['name'])
119
def validate_role_data(self, expected, actual):
120
"""Validate role data.
122
Validate a list of actual role data vs a list of expected role
125
self.log.debug('Validating role data...')
126
self.log.debug('actual: {}'.format(repr(actual)))
130
a = {'name': act.name, 'id': act.id}
131
if e['name'] == a['name']:
133
ret = self._validate_dict_data(e, a)
135
return "unexpected role data - {}".format(ret)
137
return "role {} does not exist".format(e['name'])
140
def validate_user_data(self, expected, actual):
141
"""Validate user data.
143
Validate a list of actual user data vs a list of expected user
146
self.log.debug('Validating user data...')
147
self.log.debug('actual: {}'.format(repr(actual)))
151
a = {'enabled': act.enabled, 'name': act.name,
152
'email': act.email, 'tenantId': act.tenantId,
154
if e['name'] == a['name']:
156
ret = self._validate_dict_data(e, a)
158
return "unexpected user data - {}".format(ret)
160
return "user {} does not exist".format(e['name'])
163
def validate_flavor_data(self, expected, actual):
164
"""Validate flavor data.
166
Validate a list of actual flavors vs a list of expected flavors.
168
self.log.debug('Validating flavor data...')
169
self.log.debug('actual: {}'.format(repr(actual)))
170
act = [a.name for a in actual]
171
return self._validate_list_data(expected, act)
173
def tenant_exists(self, keystone, tenant):
174
"""Return True if tenant exists."""
175
self.log.debug('Checking if tenant exists ({})...'.format(tenant))
176
return tenant in [t.name for t in keystone.tenants.list()]
178
def authenticate_cinder_admin(self, keystone_sentry, username,
180
"""Authenticates admin user with cinder."""
181
# NOTE(beisner): cinder python client doesn't accept tokens.
183
keystone_sentry.relation('shared-db',
184
'mysql:shared-db')['private-address']
185
ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
186
return cinder_client.Client(username, password, tenant, ept)
188
def authenticate_keystone_admin(self, keystone_sentry, user, password,
190
"""Authenticates admin user with the keystone admin endpoint."""
191
self.log.debug('Authenticating keystone admin...')
192
unit = keystone_sentry
193
service_ip = unit.relation('shared-db',
194
'mysql:shared-db')['private-address']
195
ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
196
return keystone_client.Client(username=user, password=password,
197
tenant_name=tenant, auth_url=ep)
199
def authenticate_keystone_user(self, keystone, user, password, tenant):
200
"""Authenticates a regular user with the keystone public endpoint."""
201
self.log.debug('Authenticating keystone user ({})...'.format(user))
202
ep = keystone.service_catalog.url_for(service_type='identity',
203
endpoint_type='publicURL')
204
return keystone_client.Client(username=user, password=password,
205
tenant_name=tenant, auth_url=ep)
207
def authenticate_glance_admin(self, keystone):
208
"""Authenticates admin user with glance."""
209
self.log.debug('Authenticating glance admin...')
210
ep = keystone.service_catalog.url_for(service_type='image',
211
endpoint_type='adminURL')
212
return glance_client.Client(ep, token=keystone.auth_token)
214
def authenticate_heat_admin(self, keystone):
215
"""Authenticates the admin user with heat."""
216
self.log.debug('Authenticating heat admin...')
217
ep = keystone.service_catalog.url_for(service_type='orchestration',
218
endpoint_type='publicURL')
219
return heat_client.Client(endpoint=ep, token=keystone.auth_token)
221
def authenticate_nova_user(self, keystone, user, password, tenant):
222
"""Authenticates a regular user with nova-api."""
223
self.log.debug('Authenticating nova user ({})...'.format(user))
224
ep = keystone.service_catalog.url_for(service_type='identity',
225
endpoint_type='publicURL')
226
return nova_client.Client(username=user, api_key=password,
227
project_id=tenant, auth_url=ep)
229
def authenticate_swift_user(self, keystone, user, password, tenant):
230
"""Authenticates a regular user with swift api."""
231
self.log.debug('Authenticating swift user ({})...'.format(user))
232
ep = keystone.service_catalog.url_for(service_type='identity',
233
endpoint_type='publicURL')
234
return swiftclient.Connection(authurl=ep,
240
def create_cirros_image(self, glance, image_name):
241
"""Download the latest cirros image and upload it to glance,
242
validate and return a resource pointer.
244
:param glance: pointer to authenticated glance connection
245
:param image_name: display name for new image
246
:returns: glance image pointer
248
self.log.debug('Creating glance cirros image '
249
'({})...'.format(image_name))
251
# Download cirros image
252
http_proxy = os.getenv('AMULET_HTTP_PROXY')
253
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
255
proxies = {'http': http_proxy}
256
opener = urllib.FancyURLopener(proxies)
258
opener = urllib.FancyURLopener()
260
f = opener.open('http://download.cirros-cloud.net/version/released')
261
version = f.read().strip()
262
cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
263
local_path = os.path.join('tests', cirros_img)
265
if not os.path.exists(local_path):
266
cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
268
opener.retrieve(cirros_url, local_path)
271
# Create glance image
272
with open(local_path) as f:
273
image = glance.images.create(name=image_name, is_public=True,
275
container_format='bare', data=f)
277
# Wait for image to reach active status
279
ret = self.resource_reaches_status(glance.images, img_id,
280
expected_stat='active',
281
msg='Image status wait')
283
msg = 'Glance image failed to reach expected state.'
284
amulet.raise_status(amulet.FAIL, msg=msg)
286
# Re-validate new image
287
self.log.debug('Validating image attributes...')
288
val_img_name = glance.images.get(img_id).name
289
val_img_stat = glance.images.get(img_id).status
290
val_img_pub = glance.images.get(img_id).is_public
291
val_img_cfmt = glance.images.get(img_id).container_format
292
val_img_dfmt = glance.images.get(img_id).disk_format
293
msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
294
'container fmt:{} disk fmt:{}'.format(
295
val_img_name, val_img_pub, img_id,
296
val_img_stat, val_img_cfmt, val_img_dfmt))
298
if val_img_name == image_name and val_img_stat == 'active' \
299
and val_img_pub is True and val_img_cfmt == 'bare' \
300
and val_img_dfmt == 'qcow2':
301
self.log.debug(msg_attr)
303
msg = ('Volume validation failed, {}'.format(msg_attr))
304
amulet.raise_status(amulet.FAIL, msg=msg)
308
def delete_image(self, glance, image):
309
"""Delete the specified image."""
311
# /!\ DEPRECATION WARNING
312
self.log.warn('/!\\ DEPRECATION WARNING: use '
313
'delete_resource instead of delete_image.')
314
self.log.debug('Deleting glance image ({})...'.format(image))
315
return self.delete_resource(glance.images, image, msg='glance image')
317
def create_instance(self, nova, image_name, instance_name, flavor):
318
"""Create the specified instance."""
319
self.log.debug('Creating instance '
320
'({}|{}|{})'.format(instance_name, image_name, flavor))
321
image = nova.images.find(name=image_name)
322
flavor = nova.flavors.find(name=flavor)
323
instance = nova.servers.create(name=instance_name, image=image,
327
status = instance.status
328
while status != 'ACTIVE' and count < 60:
330
instance = nova.servers.get(instance.id)
331
status = instance.status
332
self.log.debug('instance status: {}'.format(status))
335
if status != 'ACTIVE':
336
self.log.error('instance creation timed out')
341
def delete_instance(self, nova, instance):
342
"""Delete the specified instance."""
344
# /!\ DEPRECATION WARNING
345
self.log.warn('/!\\ DEPRECATION WARNING: use '
346
'delete_resource instead of delete_instance.')
347
self.log.debug('Deleting instance ({})...'.format(instance))
348
return self.delete_resource(nova.servers, instance,
351
def create_or_get_keypair(self, nova, keypair_name="testkey"):
352
"""Create a new keypair, or return pointer if it already exists."""
354
_keypair = nova.keypairs.get(keypair_name)
355
self.log.debug('Keypair ({}) already exists, '
356
'using it.'.format(keypair_name))
359
self.log.debug('Keypair ({}) does not exist, '
360
'creating it.'.format(keypair_name))
362
_keypair = nova.keypairs.create(name=keypair_name)
365
def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
366
img_id=None, src_vol_id=None, snap_id=None):
367
"""Create cinder volume, optionally from a glance image, OR
368
optionally as a clone of an existing volume, OR optionally
369
from a snapshot. Wait for the new volume status to reach
370
the expected status, validate and return a resource pointer.
372
:param vol_name: cinder volume display name
373
:param vol_size: size in gigabytes
374
:param img_id: optional glance image id
375
:param src_vol_id: optional source volume id to clone
376
:param snap_id: optional snapshot id to use
377
:returns: cinder volume pointer
379
# Handle parameter input and avoid impossible combinations
380
if img_id and not src_vol_id and not snap_id:
381
# Create volume from image
382
self.log.debug('Creating cinder volume from glance image...')
384
elif src_vol_id and not img_id and not snap_id:
385
# Clone an existing volume
386
self.log.debug('Cloning cinder volume...')
387
bootable = cinder.volumes.get(src_vol_id).bootable
388
elif snap_id and not src_vol_id and not img_id:
389
# Create volume from snapshot
390
self.log.debug('Creating cinder volume from snapshot...')
391
snap = cinder.volume_snapshots.find(id=snap_id)
393
snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
394
bootable = cinder.volumes.get(snap_vol_id).bootable
395
elif not img_id and not src_vol_id and not snap_id:
397
self.log.debug('Creating cinder volume...')
400
# Impossible combination of parameters
401
msg = ('Invalid method use - name:{} size:{} img_id:{} '
402
'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
405
amulet.raise_status(amulet.FAIL, msg=msg)
409
vol_new = cinder.volumes.create(display_name=vol_name,
412
source_volid=src_vol_id,
415
except Exception as e:
416
msg = 'Failed to create volume: {}'.format(e)
417
amulet.raise_status(amulet.FAIL, msg=msg)
419
# Wait for volume to reach available status
420
ret = self.resource_reaches_status(cinder.volumes, vol_id,
421
expected_stat="available",
422
msg="Volume status wait")
424
msg = 'Cinder volume failed to reach expected state.'
425
amulet.raise_status(amulet.FAIL, msg=msg)
427
# Re-validate new volume
428
self.log.debug('Validating volume attributes...')
429
val_vol_name = cinder.volumes.get(vol_id).display_name
430
val_vol_boot = cinder.volumes.get(vol_id).bootable
431
val_vol_stat = cinder.volumes.get(vol_id).status
432
val_vol_size = cinder.volumes.get(vol_id).size
433
msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
434
'{} size:{}'.format(val_vol_name, vol_id,
435
val_vol_stat, val_vol_boot,
438
if val_vol_boot == bootable and val_vol_stat == 'available' \
439
and val_vol_name == vol_name and val_vol_size == vol_size:
440
self.log.debug(msg_attr)
442
msg = ('Volume validation failed, {}'.format(msg_attr))
443
amulet.raise_status(amulet.FAIL, msg=msg)
447
def delete_resource(self, resource, resource_id,
448
msg="resource", max_wait=120):
449
"""Delete one openstack resource, such as one instance, keypair,
450
image, volume, stack, etc., and confirm deletion within max wait time.
452
:param resource: pointer to os resource type, ex:glance_client.images
453
:param resource_id: unique name or id for the openstack resource
454
:param msg: text to identify purpose in logging
455
:param max_wait: maximum wait time in seconds
456
:returns: True if successful, otherwise False
458
self.log.debug('Deleting OpenStack resource '
459
'{} ({})'.format(resource_id, msg))
460
num_before = len(list(resource.list()))
461
resource.delete(resource_id)
464
num_after = len(list(resource.list()))
465
while num_after != (num_before - 1) and tries < (max_wait / 4):
466
self.log.debug('{} delete check: '
467
'{} [{}:{}] {}'.format(msg, tries,
472
num_after = len(list(resource.list()))
475
self.log.debug('{}: expected, actual count = {}, '
476
'{}'.format(msg, num_before - 1, num_after))
478
if num_after == (num_before - 1):
481
self.log.error('{} delete timed out'.format(msg))
484
def resource_reaches_status(self, resource, resource_id,
485
expected_stat='available',
486
msg='resource', max_wait=120):
487
"""Wait for an openstack resources status to reach an
488
expected status within a specified time. Useful to confirm that
489
nova instances, cinder vols, snapshots, glance images, heat stacks
490
and other resources eventually reach the expected status.
492
:param resource: pointer to os resource type, ex: heat_client.stacks
493
:param resource_id: unique id for the openstack resource
494
:param expected_stat: status to expect resource to reach
495
:param msg: text to identify purpose in logging
496
:param max_wait: maximum wait time in seconds
497
:returns: True if successful, False if status is not reached
501
resource_stat = resource.get(resource_id).status
502
while resource_stat != expected_stat and tries < (max_wait / 4):
503
self.log.debug('{} status check: '
504
'{} [{}:{}] {}'.format(msg, tries,
509
resource_stat = resource.get(resource_id).status
512
self.log.debug('{}: expected, actual status = {}, '
513
'{}'.format(msg, resource_stat, expected_stat))
515
if resource_stat == expected_stat:
518
self.log.debug('{} never reached expected status: '
519
'{}'.format(resource_id, expected_stat))
522
def get_ceph_osd_id_cmd(self, index):
523
"""Produce a shell command that will return a ceph-osd id."""
524
return ("`initctl list | grep 'ceph-osd ' | "
525
"awk 'NR=={} {{ print $2 }}' | "
526
"grep -o '[0-9]*'`".format(index + 1))
528
def get_ceph_pools(self, sentry_unit):
529
"""Return a dict of ceph pools from a single ceph unit, with
530
pool name as keys, pool id as vals."""
532
cmd = 'sudo ceph osd lspools'
533
output, code = sentry_unit.run(cmd)
535
msg = ('{} `{}` returned {} '
536
'{}'.format(sentry_unit.info['unit_name'],
538
amulet.raise_status(amulet.FAIL, msg=msg)
540
# Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
541
for pool in str(output).split(','):
542
pool_id_name = pool.split(' ')
543
if len(pool_id_name) == 2:
544
pool_id = pool_id_name[0]
545
pool_name = pool_id_name[1]
546
pools[pool_name] = int(pool_id)
548
self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
552
def get_ceph_df(self, sentry_unit):
553
"""Return dict of ceph df json output, including ceph pool state.
555
:param sentry_unit: Pointer to amulet sentry instance (juju unit)
556
:returns: Dict of ceph df output
558
cmd = 'sudo ceph df --format=json'
559
output, code = sentry_unit.run(cmd)
561
msg = ('{} `{}` returned {} '
562
'{}'.format(sentry_unit.info['unit_name'],
564
amulet.raise_status(amulet.FAIL, msg=msg)
565
return json.loads(output)
567
def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
568
"""Take a sample of attributes of a ceph pool, returning ceph
569
pool name, object count and disk space used for the specified
572
:param sentry_unit: Pointer to amulet sentry instance (juju unit)
573
:param pool_id: Ceph pool ID
574
:returns: List of pool name, object count, kb disk space used
576
df = self.get_ceph_df(sentry_unit)
577
pool_name = df['pools'][pool_id]['name']
578
obj_count = df['pools'][pool_id]['stats']['objects']
579
kb_used = df['pools'][pool_id]['stats']['kb_used']
580
self.log.debug('Ceph {} pool (ID {}): {} objects, '
581
'{} kb used'.format(pool_name, pool_id,
583
return pool_name, obj_count, kb_used
585
def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
586
"""Validate ceph pool samples taken over time, such as pool
587
object counts or pool kb used, before adding, after adding, and
588
after deleting items which affect those pool attributes. The
589
2nd element is expected to be greater than the 1st; 3rd is expected
590
to be less than the 2nd.
592
:param samples: List containing 3 data samples
593
:param sample_type: String for logging and usage context
594
:returns: None if successful, Failure message otherwise
596
original, created, deleted = range(3)
597
if samples[created] <= samples[original] or \
598
samples[deleted] >= samples[created]:
599
return ('Ceph {} samples ({}) '
600
'unexpected.'.format(sample_type, samples))
602
self.log.debug('Ceph {} samples (OK): '
603
'{}'.format(sample_type, samples))
added
removed
hooks/charmhelpers/contrib/openstack/amulet/utils.py
