1
# Copyright 2016 Canonical Ltd
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
7
# http://www.apache.org/licenses/LICENSE-2.0
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
15
from collections import OrderedDict
16
from copy import deepcopy
17
from functools import partial
22
from base64 import b64encode
23
from charmhelpers.contrib.openstack import context, templating
24
from charmhelpers.contrib.openstack.neutron import (
25
neutron_plugin_attribute,
28
from charmhelpers.contrib.openstack.utils import (
30
get_os_codename_install_source,
31
git_clone_and_install,
33
git_generate_systemd_init_files,
34
git_install_requested,
38
configure_installation_source,
39
incomplete_relation_data,
41
make_assess_status_func,
44
os_application_version_set,
49
from charmhelpers.contrib.python.packages import (
53
from charmhelpers.core.hookenv import (
60
from charmhelpers.fetch import (
67
from charmhelpers.core.host import (
79
from charmhelpers.contrib.hahelpers.cluster import (
84
from charmhelpers.core.templating import render
85
from charmhelpers.contrib.hahelpers.cluster import is_elected_leader
87
import neutron_api_context
89
TEMPLATES = 'templates/'
91
CLUSTER_RES = 'grp_neutron_vips'
93
# removed from original: charm-helper-sh
97
'python-keystoneclient',
105
'python-neutron-lbaas',
106
'python-neutron-fwaas',
107
'python-neutron-vpnaas',
110
VERSION_PACKAGE = 'neutron-common'
112
BASE_GIT_PACKAGES = [
114
'libmysqlclient-dev',
119
'openstack-pkg-tools',
121
'python-neutronclient', # required for get_neutron_client() import
127
# ubuntu packages that should not be installed when deploying from git
128
GIT_PACKAGE_BLACKLIST = [
130
'neutron-plugin-ml2',
131
'python-keystoneclient',
135
GIT_PACKAGE_BLACKLIST_KILO = [
136
'python-neutron-lbaas',
137
'python-neutron-fwaas',
138
'python-neutron-vpnaas',
145
'neutron-server': 9696,
148
NEUTRON_CONF_DIR = "/etc/neutron"
150
NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR
151
NEUTRON_LBAAS_CONF = '%s/neutron_lbaas.conf' % NEUTRON_CONF_DIR
152
NEUTRON_VPNAAS_CONF = '%s/neutron_vpnaas.conf' % NEUTRON_CONF_DIR
153
HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
154
APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend'
155
APACHE_24_CONF = '/etc/apache2/sites-available/openstack_https_frontend.conf'
156
NEUTRON_DEFAULT = '/etc/default/neutron-server'
157
CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
158
MEMCACHED_CONF = '/etc/memcached.conf'
159
API_PASTE_INI = '%s/api-paste.ini' % NEUTRON_CONF_DIR
161
BASE_RESOURCE_MAP = OrderedDict([
163
'services': ['neutron-server'],
164
'contexts': [context.AMQPContext(ssl_dir=NEUTRON_CONF_DIR),
165
context.SharedDBContext(
166
user=config('database-user'),
167
database=config('database'),
168
ssl_dir=NEUTRON_CONF_DIR),
169
context.PostgresqlDBContext(database=config('database')),
170
neutron_api_context.IdentityServiceContext(
172
service_user='neutron'),
173
context.OSConfigFlagContext(),
174
neutron_api_context.NeutronCCContext(),
175
context.SyslogContext(),
176
context.ZeroMQContext(),
177
context.NotificationDriverContext(),
178
context.BindHostContext(),
179
context.WorkerConfigContext(),
180
context.InternalEndpointContext(),
181
context.MemcacheContext()],
184
'services': ['neutron-server'],
185
'contexts': [neutron_api_context.NeutronCCContext()],
188
'services': ['neutron-server'],
189
'contexts': [neutron_api_context.NeutronApiApiPasteContext()],
192
'contexts': [neutron_api_context.ApacheSSLContext()],
193
'services': ['apache2'],
196
'contexts': [neutron_api_context.ApacheSSLContext()],
197
'services': ['apache2'],
200
'contexts': [context.HAProxyContext(singlenode_mode=True),
201
neutron_api_context.HAProxyContext()],
202
'services': ['haproxy'],
206
# The interface is said to be satisfied if anyone of the interfaces in the
207
# list has a complete context.
208
REQUIRED_INTERFACES = {
209
'database': ['shared-db', 'pgsql-db'],
210
'messaging': ['amqp', 'zeromq-configuration'],
211
'identity': ['identity-service'],
214
LIBERTY_RESOURCE_MAP = OrderedDict([
215
(NEUTRON_LBAAS_CONF, {
216
'services': ['neutron-server'],
219
(NEUTRON_VPNAAS_CONF, {
220
'services': ['neutron-server'],
226
def api_port(service):
227
return API_PORTS[service]
230
def additional_install_locations(plugin, source):
232
Add any required additional package locations for the charm, based
233
on the Neutron plugin being used. This will also force an immediate
236
release = get_os_codename_install_source(source)
237
if plugin == 'Calico':
238
if config('calico-origin'):
239
calico_source = config('calico-origin')
240
elif release in ('icehouse', 'juno', 'kilo'):
241
# Prior to the Liberty release, Calico's Nova and Neutron changes
242
# were not fully upstreamed, so we need to point to a
243
# release-specific PPA that includes Calico-specific Nova and
245
calico_source = 'ppa:project-calico/%s' % release
247
# From Liberty onwards, we can point to a PPA that does not include
248
# any patched OpenStack packages, and hence is independent of the
250
calico_source = 'ppa:project-calico/calico-1.4'
252
add_source(calico_source)
254
elif plugin == 'midonet':
255
midonet_origin = config('midonet-origin')
256
release_num = midonet_origin.split('-')[1]
258
if midonet_origin.startswith('mem'):
259
with open(os.path.join(charm_dir(),
260
'files/midokura.key')) as midokura_gpg_key:
261
priv_gpg_key = midokura_gpg_key.read()
262
mem_username = config('mem-username')
263
mem_password = config('mem-password')
264
if release in ('juno', 'kilo', 'liberty'):
266
'deb http://%s:%s@apt.midokura.com/openstack/%s/stable '
267
'trusty main' % (mem_username, mem_password, release),
269
add_source('http://%s:%s@apt.midokura.com/midonet/v%s/stable '
270
'main' % (mem_username, mem_password, release_num),
273
with open(os.path.join(charm_dir(),
274
'files/midonet.key')) as midonet_gpg_key:
275
pub_gpg_key = midonet_gpg_key.read()
276
if release in ('juno', 'kilo', 'liberty'):
278
'deb http://repo.midonet.org/openstack-%s stable main' %
279
release, key=pub_gpg_key)
281
add_source('deb http://repo.midonet.org/midonet/v%s stable main' %
282
release_num, key=pub_gpg_key)
284
apt_update(fatal=True)
285
apt_upgrade(fatal=True)
288
def force_etcd_restart():
290
If etcd has been reconfigured we need to force it to fully restart.
291
This is necessary because etcd has some config flags that it ignores
292
after the first time it starts, so we need to make it forget them.
295
for directory in glob.glob('/var/lib/etcd/*'):
296
shutil.rmtree(directory)
297
if not is_unit_paused_set():
298
service_start('etcd')
302
return config('manage-neutron-plugin-legacy-mode')
305
def determine_packages(source=None):
306
# currently all packages match service names
307
packages = [] + BASE_PACKAGES
309
for v in resource_map().values():
310
packages.extend(v['services'])
312
pkgs = neutron_plugin_attribute(config('neutron-plugin'),
315
packages.extend(pkgs)
317
release = get_os_codename_install_source(source)
319
if release >= 'kilo':
320
packages.extend(KILO_PACKAGES)
322
if release == 'kilo' or release >= 'mitaka':
323
packages.append('python-networking-hyperv')
325
if config('neutron-plugin') == 'vsp':
326
nuage_pkgs = config('nuage-packages').split()
327
packages += nuage_pkgs
329
if git_install_requested():
330
packages.extend(BASE_GIT_PACKAGES)
331
# don't include packages that will be installed from git
332
packages = list(set(packages))
333
for p in GIT_PACKAGE_BLACKLIST:
336
if release >= 'kilo':
337
for p in GIT_PACKAGE_BLACKLIST_KILO:
340
packages.extend(token_cache_pkgs(release=release))
341
return list(set(packages))
344
def determine_ports():
345
'''Assemble a list of API ports for services we are managing'''
347
for services in restart_map().values():
348
for service in services:
350
ports.append(API_PORTS[service])
353
return list(set(ports))
356
def resource_map(release=None):
358
Dynamically generate a map of resources that will be managed for a single
361
release = release or os_release('neutron-common')
363
resource_map = deepcopy(BASE_RESOURCE_MAP)
364
if release >= 'liberty':
365
resource_map.update(LIBERTY_RESOURCE_MAP)
367
if os.path.exists('/etc/apache2/conf-available'):
368
resource_map.pop(APACHE_CONF)
370
resource_map.pop(APACHE_24_CONF)
373
# add neutron plugin requirements. nova-c-c only needs the
374
# neutron-server associated with configs, not the plugin agent.
375
plugin = config('neutron-plugin')
376
conf = neutron_plugin_attribute(plugin, 'config', 'neutron')
377
ctxts = (neutron_plugin_attribute(plugin, 'contexts', 'neutron') or
379
services = neutron_plugin_attribute(plugin, 'server_services',
381
resource_map[conf] = {}
382
resource_map[conf]['services'] = services
383
resource_map[conf]['contexts'] = ctxts
384
resource_map[conf]['contexts'].append(
385
neutron_api_context.NeutronCCContext())
387
# update for postgres
388
resource_map[conf]['contexts'].append(
389
context.PostgresqlDBContext(database=config('database')))
392
resource_map[NEUTRON_CONF]['contexts'].append(
393
neutron_api_context.NeutronApiSDNContext()
395
resource_map[NEUTRON_DEFAULT]['contexts'] = \
396
[neutron_api_context.NeutronApiSDNConfigFileContext()]
397
if enable_memcache(release=release):
398
resource_map[MEMCACHED_CONF] = {
399
'contexts': [context.MemcacheContext()],
400
'services': ['memcached']}
405
def register_configs(release=None):
406
release = release or os_release('neutron-common')
407
configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
408
openstack_release=release)
409
for cfg, rscs in resource_map().iteritems():
410
configs.register(cfg, rscs['contexts'])
415
return OrderedDict([(cfg, v['services'])
416
for cfg, v in resource_map().iteritems()
421
''' Returns a list of services associate with this charm '''
423
for v in restart_map().values():
424
_services = _services + v
425
return list(set(_services))
428
def keystone_ca_cert_b64():
429
'''Returns the local Keystone-provided CA cert if it exists, or None.'''
430
if not os.path.isfile(CA_CERT_PATH):
432
with open(CA_CERT_PATH) as _in:
433
return b64encode(_in.read())
436
def do_openstack_upgrade(configs):
438
Perform an upgrade. Takes care of upgrading packages, rewriting
439
configs, database migrations and potentially any other post-upgrade
442
:param configs: The charms main OSConfigRenderer object.
444
cur_os_rel = os_release('neutron-common')
445
new_src = config('openstack-origin')
446
new_os_rel = get_os_codename_install_source(new_src)
448
log('Performing OpenStack upgrade to %s.' % (new_os_rel))
450
configure_installation_source(new_src)
452
'--option', 'Dpkg::Options::=--force-confnew',
453
'--option', 'Dpkg::Options::=--force-confdef',
455
apt_update(fatal=True)
456
apt_upgrade(options=dpkg_opts, fatal=True, dist=True)
457
pkgs = determine_packages(new_os_rel)
458
# Sort packages just to make unit tests easier
460
apt_install(packages=pkgs,
464
# set CONFIGS to load templates from new release
465
configs.set_release(openstack_release=new_os_rel)
466
# Before kilo it's nova-cloud-controllers job
467
if is_elected_leader(CLUSTER_RES):
468
# Stamping seems broken and unnecessary in liberty (Bug #1536675)
469
if os_release('neutron-common') < 'liberty':
470
stamp_neutron_database(cur_os_rel)
471
migrate_neutron_database()
474
def stamp_neutron_database(release):
475
'''Stamp the database with the current release before upgrade.'''
476
log('Stamping the neutron database with release %s.' % release)
477
plugin = config('neutron-plugin')
478
cmd = ['neutron-db-manage',
479
'--config-file', NEUTRON_CONF,
480
'--config-file', neutron_plugin_attribute(plugin,
485
subprocess.check_output(cmd)
488
def nuage_vsp_juno_neutron_migration():
489
log('Nuage VSP with Juno Relase')
490
nuage_migration_db_path = '/usr/lib/python2.7/dist-packages/'\
491
'neutron/db/migration/nuage'
492
nuage_migrate_hybrid_file_path = os.path.join(
493
nuage_migration_db_path, 'migrate_hybrid_juno.py')
494
nuage_config_file = neutron_plugin_attribute(config('neutron-plugin'),
496
if os.path.exists(nuage_migration_db_path):
497
if os.path.exists(nuage_migrate_hybrid_file_path):
498
if os.path.exists(nuage_config_file):
499
log('Running Migartion Script for Juno Release')
500
cmd = 'sudo python ' + nuage_migrate_hybrid_file_path + \
501
' --config-file ' + nuage_config_file + \
502
' --config-file ' + NEUTRON_CONF
504
subprocess.check_output(cmd, shell=True)
506
e = nuage_config_file+' doesnot exist'
510
e = nuage_migrate_hybrid_file_path+' doesnot exists'
514
e = nuage_migration_db_path+' doesnot exists'
519
def migrate_neutron_database():
520
'''Initializes a new database or upgrades an existing database.'''
521
log('Migrating the neutron database.')
522
if(os_release('neutron-server') == 'juno' and
523
config('neutron-plugin') == 'vsp'):
524
nuage_vsp_juno_neutron_migration()
526
plugin = config('neutron-plugin')
527
cmd = ['neutron-db-manage',
528
'--config-file', NEUTRON_CONF,
529
'--config-file', neutron_plugin_attribute(plugin,
534
subprocess.check_output(cmd)
538
return ['q-l3-plugin',
548
ubuntu_rel = lsb_release()['DISTRIB_CODENAME'].lower()
549
if ubuntu_rel < "trusty":
550
raise Exception("IPv6 is not supported in the charms for Ubuntu "
551
"versions less than Trusty 14.04")
553
# Need haproxy >= 1.5.3 for ipv6 so for Trusty if we are <= Kilo we need to
554
# use trusty-backports otherwise we can use the UCA.
555
if ubuntu_rel == 'trusty' and os_release('neutron-server') < 'liberty':
556
add_source('deb http://archive.ubuntu.com/ubuntu trusty-backports '
559
apt_install('haproxy/trusty-backports', fatal=True)
562
def get_neutron_client():
563
''' Return a neutron client if possible '''
564
env = neutron_api_context.IdentityServiceContext()()
566
log('Unable to check resources at this time')
569
auth_url = '%(auth_protocol)s://%(auth_host)s:%(auth_port)s/v2.0' % env
570
# Late import to avoid install hook failures when pkg hasnt been installed
571
from neutronclient.v2_0 import client
572
neutron_client = client.Client(username=env['admin_user'],
573
password=env['admin_password'],
574
tenant_name=env['admin_tenant_name'],
576
region_name=env['region'])
577
return neutron_client
580
def router_feature_present(feature):
581
''' Check For dvr enabled routers '''
582
neutron_client = get_neutron_client()
583
for router in neutron_client.list_routers()['routers']:
584
if router.get(feature, False):
588
l3ha_router_present = partial(router_feature_present, feature='ha')
590
dvr_router_present = partial(router_feature_present, feature='distributed')
594
''' Check if neutron is ready by running arbitrary query'''
595
neutron_client = get_neutron_client()
596
if not neutron_client:
597
log('No neutron client, neutron not ready')
600
neutron_client.list_routers()
601
log('neutron client ready')
604
log('neutron query failed, neutron not ready ')
608
def git_install(projects_yaml):
609
"""Perform setup, and install git repos specified in yaml parameter."""
610
if git_install_requested():
612
projects_yaml = git_default_repos(projects_yaml)
613
git_clone_and_install(projects_yaml, core_project='neutron')
614
git_post_install(projects_yaml)
617
def git_pre_install():
618
"""Perform pre-install setup."""
621
'/var/lib/neutron/lock',
626
'/var/log/neutron/server.log',
629
adduser('neutron', shell='/bin/bash', system_user=True)
630
add_group('neutron', system_group=True)
631
add_user_to_group('neutron', 'neutron')
634
mkdir(d, owner='neutron', group='neutron', perms=0755, force=False)
637
write_file(l, '', owner='neutron', group='neutron', perms=0600)
640
def git_post_install(projects_yaml):
641
"""Perform post-install setup."""
642
http_proxy = git_yaml_value(projects_yaml, 'http_proxy')
644
pip_install('mysql-python', proxy=http_proxy,
645
venv=git_pip_venv_dir(projects_yaml))
647
pip_install('mysql-python',
648
venv=git_pip_venv_dir(projects_yaml))
650
src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc')
653
'dest': '/etc/neutron'},
654
{'src': os.path.join(src_etc, 'neutron/plugins'),
655
'dest': '/etc/neutron/plugins'},
656
{'src': os.path.join(src_etc, 'neutron/rootwrap.d'),
657
'dest': '/etc/neutron/rootwrap.d'},
661
if os.path.exists(c['dest']):
662
shutil.rmtree(c['dest'])
663
shutil.copytree(c['src'], c['dest'])
665
# NOTE(coreycb): Need to find better solution than bin symlinks.
667
{'src': os.path.join(git_pip_venv_dir(projects_yaml),
668
'bin/neutron-rootwrap'),
669
'link': '/usr/local/bin/neutron-rootwrap'},
670
{'src': os.path.join(git_pip_venv_dir(projects_yaml),
671
'bin/neutron-db-manage'),
672
'link': '/usr/local/bin/neutron-db-manage'},
676
if os.path.lexists(s['link']):
678
os.symlink(s['src'], s['link'])
680
render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {},
683
bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
684
# Use systemd init units/scripts from ubuntu wily onward
685
if lsb_release()['DISTRIB_RELEASE'] >= '15.10':
686
templates_dir = os.path.join(charm_dir(), 'templates/git')
687
daemon = 'neutron-server'
688
neutron_api_context = {
689
'daemon_path': os.path.join(bin_dir, daemon),
691
template_file = 'git/{}.init.in.template'.format(daemon)
692
init_in_file = '{}.init.in'.format(daemon)
693
render(template_file, os.path.join(templates_dir, init_in_file),
694
neutron_api_context, perms=0o644)
695
git_generate_systemd_init_files(templates_dir)
697
neutron_api_context = {
698
'service_description': 'Neutron API server',
699
'charm_name': 'neutron-api',
700
'process_name': 'neutron-server',
701
'executable_name': os.path.join(bin_dir, 'neutron-server'),
704
render('git/upstart/neutron-server.upstart',
705
'/etc/init/neutron-server.conf',
706
neutron_api_context, perms=0o644)
708
if not is_unit_paused_set():
709
service_restart('neutron-server')
712
def get_optional_interfaces():
713
"""Return the optional interfaces that should be checked if the relavent
714
relations have appeared.
715
:returns: {general_interface: [specific_int1, specific_int2, ...], ...}
717
optional_interfaces = {}
718
if relation_ids('ha'):
719
optional_interfaces['ha'] = ['cluster']
720
return optional_interfaces
723
def check_optional_relations(configs):
724
"""Check that if we have a relation_id for high availability that we can
725
get the hacluster config. If we can't then we are blocked. This function
726
is called from assess_status/set_os_workload_status as the charm_func and
727
needs to return either "unknown", "" if there is no problem or the status,
728
message if there is a problem.
730
:param configs: an OSConfigRender() instance.
731
:return 2-tuple: (string, string) = (status, message)
733
if relation_ids('ha'):
735
get_hacluster_config()
738
'hacluster missing configuration: '
739
'vip, vip_iface, vip_cidr')
740
# return 'unknown' as the lowest priority to not clobber an existing
745
def is_api_ready(configs):
746
return (not incomplete_relation_data(configs, REQUIRED_INTERFACES))
749
def assess_status(configs):
750
"""Assess status of current unit
751
Decides what the state of the unit should be based on the current
753
SIDE EFFECT: calls set_os_workload_status(...) which sets the workload
755
Also calls status_set(...) directly if paused state isn't complete.
756
@param configs: a templating.OSConfigRenderer() object
757
@returns None - this function is executed for its side-effect
759
assess_status_func(configs)()
760
os_application_version_set(VERSION_PACKAGE)
763
def assess_status_func(configs):
764
"""Helper function to create the function that will assess_status() for
766
Uses charmhelpers.contrib.openstack.utils.make_assess_status_func() to
767
create the appropriate status function and then returns it.
768
Used directly by assess_status() and also for pausing and resuming
771
NOTE: REQUIRED_INTERFACES is augmented with the optional interfaces
772
depending on the current config before being passed to the
773
make_assess_status_func() function.
775
NOTE(ajkavanagh) ports are not checked due to race hazards with services
776
that don't behave sychronously w.r.t their service scripts. e.g.
779
@param configs: a templating.OSConfigRenderer() object
780
@return f() -> None : a function that assesses the unit's workload status
782
required_interfaces = REQUIRED_INTERFACES.copy()
783
required_interfaces.update(get_optional_interfaces())
784
return make_assess_status_func(
785
configs, required_interfaces,
786
charm_func=check_optional_relations,
787
services=services(), ports=None)
790
def pause_unit_helper(configs):
791
"""Helper function to pause a unit, and then call assess_status(...) in
792
effect, so that the status is correctly updated.
793
Uses charmhelpers.contrib.openstack.utils.pause_unit() to do the work.
794
@param configs: a templating.OSConfigRenderer() object
795
@returns None - this function is executed for its side-effect
797
_pause_resume_helper(pause_unit, configs)
800
def resume_unit_helper(configs):
801
"""Helper function to resume a unit, and then call assess_status(...) in
802
effect, so that the status is correctly updated.
803
Uses charmhelpers.contrib.openstack.utils.resume_unit() to do the work.
804
@param configs: a templating.OSConfigRenderer() object
805
@returns None - this function is executed for its side-effect
807
_pause_resume_helper(resume_unit, configs)
810
def _pause_resume_helper(f, configs):
811
"""Helper function that uses the make_assess_status_func(...) from
812
charmhelpers.contrib.openstack.utils to create an assess_status(...)
813
function that can be used with the pause/resume of the unit
814
@param f: the function to be used with the assess_status(...) function
815
@returns None - this function is executed for its side-effect
817
# TODO(ajkavanagh) - ports= has been left off because of the race hazard
818
# that exists due to service_start()
819
f(assess_status_func(configs),
added
removed
hooks/neutron_api_utils.py
