« back to all changes in this revision
Viewing changes to hooks/keystone_utils.py
- browse files at revision 109
- compare with another revision
- download diff
- download tarball
- view history from revision 109
- Committer: Edward Hope-Morley
- Date: 2015-01-22 14:35:41 UTC
- mfrom: (96.1.17 keystone.fix-cert-sync)
- Revision ID: edward.hope-morley@canonical.com-20150122143541-yyhs3f39w7066wdv
[hopem,r=gnuoy]
Fixes ssl cert synchronisation across peers
Closes-Bug: 1317782
Fixes ssl cert synchronisation across peers
Closes-Bug: 1317782
added
removed
hooks/keystone_utils.py
1
1
#!/usr/bin/python
2
import glob
3
import grp
4
import hashlib
5
import json
6
import os
7
import pwd
8
import re
2
9
import subprocess
3
import os
10
import threading
11
import time
12
import urlparse
4
13
import uuid
5
import urlparse
6
import time
7
14
8
15
from base64 import b64encode
9
16
from collections import OrderedDict
10
17
from copy import deepcopy
11
18
12
19
from charmhelpers.contrib.hahelpers.cluster import(
13
eligible_leader,
20
is_elected_leader,
14
21
determine_api_port,
15
22
https,
16
is_clustered,
17
is_elected_leader,
23
peer_units,
24
oldest_peer,
18
25
)
19
26
20
27
from charmhelpers.contrib.openstack import context, templating
37
44
os_release,
38
45
save_script_rc as _save_script_rc)
39
46
47
from charmhelpers.core.host import (
48
mkdir,
49
write_file,
50
)
51
40
52
import charmhelpers.contrib.unison as unison
41
53
54
from charmhelpers.core.decorators import (
55
retry_on_exception,
56
)
57
42
58
from charmhelpers.core.hookenv import (
43
59
config,
44
60
is_relation_made,
47
63
relation_get,
48
64
relation_set,
49
65
relation_ids,
66
related_units,
50
67
DEBUG,
51
68
INFO,
69
WARNING,
70
ERROR,
52
71
)
53
72
54
73
from charmhelpers.fetch import (
61
80
from charmhelpers.core.host import (
62
81
service_stop,
63
82
service_start,
83
service_restart,
64
84
pwgen,
65
85
lsb_release
66
86
)
110
130
APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend'
111
131
APACHE_24_CONF = '/etc/apache2/sites-available/openstack_https_frontend.conf'
112
132
133
APACHE_SSL_DIR = '/etc/apache2/ssl/keystone'
134
SYNC_FLAGS_DIR = '/var/lib/keystone/juju_sync_flags/'
113
135
SSL_DIR = '/var/lib/keystone/juju_ssl/'
114
136
SSL_CA_NAME = 'Ubuntu Cloud'
115
137
CLUSTER_RES = 'grp_ks_vips'
116
138
SSH_USER = 'juju_keystone'
139
SSL_SYNC_SEMAPHORE = threading.Semaphore()
117
140
118
141
BASE_RESOURCE_MAP = OrderedDict([
119
142
(KEYSTONE_CONF, {
203
226
}
204
227
205
228
229
def is_str_true(value):
230
if value and value.lower() in ['true', 'yes']:
231
return True
232
233
return False
234
235
206
236
def resource_map():
207
237
'''
208
238
Dynamically generate a map of resources that will be managed for a single
287
317
configs.set_release(openstack_release=new_os_rel)
288
318
configs.write_all()
289
319
290
if eligible_leader(CLUSTER_RES):
320
if is_elected_leader(CLUSTER_RES):
291
321
migrate_database()
292
322
293
323
389
419
390
420
up_to_date = True
391
421
for k in ['publicurl', 'adminurl', 'internalurl']:
392
if ep[k] != locals()[k]:
422
if ep.get(k) != locals()[k]:
393
423
up_to_date = False
394
424
395
425
if up_to_date:
500
530
if passwd and passwd.lower() != "none":
501
531
return passwd
502
532
503
if eligible_leader(CLUSTER_RES):
533
if is_elected_leader(CLUSTER_RES):
504
534
if os.path.isfile(STORED_PASSWD):
505
535
log("Loading stored passwd from %s" % STORED_PASSWD, level=INFO)
506
536
with open(STORED_PASSWD, 'r') as fd:
527
557
528
558
529
559
def ensure_initial_admin(config):
530
""" Ensures the minimum admin stuff exists in whatever database we're
560
# Allow retry on fail since leader may not be ready yet.
561
# NOTE(hopem): ks client may not be installed at module import time so we
562
# use this wrapped approach instead.
563
from keystoneclient.apiclient.exceptions import InternalServerError
564
565
@retry_on_exception(3, base_delay=3, exc_type=InternalServerError)
566
def _ensure_initial_admin(config):
567
"""Ensures the minimum admin stuff exists in whatever database we're
531
568
using.
569
532
570
This and the helper functions it calls are meant to be idempotent and
533
571
run during install as well as during db-changed. This will maintain
534
572
the admin tenant, user, role, service entry and endpoint across every
535
573
datastore we might use.
574
536
575
TODO: Possibly migrate data from one backend to another after it
537
576
changes?
538
"""
539
create_tenant("admin")
540
create_tenant(config("service-tenant"))
541
# User is managed by ldap backend when using ldap identity
542
if not (config('identity-backend') == 'ldap' and config('ldap-readonly')):
543
passwd = get_admin_passwd()
544
if passwd:
545
create_user(config('admin-user'), passwd, tenant='admin')
546
update_user_password(config('admin-user'), passwd)
547
create_role(config('admin-role'), config('admin-user'), 'admin')
548
create_service_entry("keystone", "identity", "Keystone Identity Service")
549
550
for region in config('region').split():
551
create_keystone_endpoint(public_ip=resolve_address(PUBLIC),
552
service_port=config("service-port"),
553
internal_ip=resolve_address(INTERNAL),
554
admin_ip=resolve_address(ADMIN),
555
auth_port=config("admin-port"),
556
region=region)
577
"""
578
create_tenant("admin")
579
create_tenant(config("service-tenant"))
580
# User is managed by ldap backend when using ldap identity
581
if not (config('identity-backend') ==
582
'ldap' and config('ldap-readonly')):
583
passwd = get_admin_passwd()
584
if passwd:
585
create_user(config('admin-user'), passwd, tenant='admin')
586
update_user_password(config('admin-user'), passwd)
587
create_role(config('admin-role'), config('admin-user'),
588
'admin')
589
create_service_entry("keystone", "identity",
590
"Keystone Identity Service")
591
592
for region in config('region').split():
593
create_keystone_endpoint(public_ip=resolve_address(PUBLIC),
594
service_port=config("service-port"),
595
internal_ip=resolve_address(INTERNAL),
596
admin_ip=resolve_address(ADMIN),
597
auth_port=config("admin-port"),
598
region=region)
599
600
return _ensure_initial_admin(config)
557
601
558
602
559
603
def endpoint_url(ip, port):
621
665
return passwd
622
666
623
667
624
def synchronize_ca():
625
'''
626
Broadcast service credentials to peers or consume those that have been
627
broadcasted by peer, depending on hook context.
628
'''
629
if not eligible_leader(CLUSTER_RES):
630
return
631
log('Synchronizing CA to all peers.')
632
if is_clustered():
633
if config('https-service-endpoints') in ['True', 'true']:
634
unison.sync_to_peers(peer_interface='cluster',
635
paths=[SSL_DIR], user=SSH_USER, verbose=True)
636
637
CA = []
668
def ensure_permissions(path, user=None, group=None, perms=None):
669
"""Set chownand chmod for path
670
671
Note that -1 for uid or gid result in no change.
672
"""
673
if user:
674
uid = pwd.getpwnam(user).pw_uid
675
else:
676
uid = -1
677
678
if group:
679
gid = grp.getgrnam(group).gr_gid
680
else:
681
gid = -1
682
683
os.chown(path, uid, gid)
684
685
if perms:
686
os.chmod(path, perms)
687
688
689
def check_peer_actions():
690
"""Honour service action requests from sync master.
691
692
Check for service action request flags, perform the action then delete the
693
flag.
694
"""
695
restart = relation_get(attribute='restart-services-trigger')
696
if restart and os.path.isdir(SYNC_FLAGS_DIR):
697
for flagfile in glob.glob(os.path.join(SYNC_FLAGS_DIR, '*')):
698
flag = os.path.basename(flagfile)
699
key = re.compile("^(.+)?\.(.+)?\.(.+)")
700
res = re.search(key, flag)
701
if res:
702
source = res.group(1)
703
service = res.group(2)
704
action = res.group(3)
705
else:
706
key = re.compile("^(.+)?\.(.+)?")
707
res = re.search(key, flag)
708
source = res.group(1)
709
action = res.group(2)
710
711
# Don't execute actions requested by this unit.
712
if local_unit().replace('.', '-') != source:
713
if action == 'restart':
714
log("Running action='%s' on service '%s'" %
715
(action, service), level=DEBUG)
716
service_restart(service)
717
elif action == 'start':
718
log("Running action='%s' on service '%s'" %
719
(action, service), level=DEBUG)
720
service_start(service)
721
elif action == 'stop':
722
log("Running action='%s' on service '%s'" %
723
(action, service), level=DEBUG)
724
service_stop(service)
725
elif action == 'update-ca-certificates':
726
log("Running %s" % (action), level=DEBUG)
727
subprocess.check_call(['update-ca-certificates'])
728
else:
729
log("Unknown action flag=%s" % (flag), level=WARNING)
730
731
try:
732
os.remove(flagfile)
733
except:
734
pass
735
736
737
def create_peer_service_actions(action, services):
738
"""Mark remote services for action.
739
740
Default action is restart. These action will be picked up by peer units
741
e.g. we may need to restart services on peer units after certs have been
742
synced.
743
"""
744
for service in services:
745
flagfile = os.path.join(SYNC_FLAGS_DIR, '%s.%s.%s' %
746
(local_unit().replace('/', '-'),
747
service.strip(), action))
748
log("Creating action %s" % (flagfile), level=DEBUG)
749
write_file(flagfile, content='', owner=SSH_USER, group='keystone',
750
perms=0o644)
751
752
753
def create_peer_actions(actions):
754
for action in actions:
755
action = "%s.%s" % (local_unit().replace('/', '-'), action)
756
flagfile = os.path.join(SYNC_FLAGS_DIR, action)
757
log("Creating action %s" % (flagfile), level=DEBUG)
758
write_file(flagfile, content='', owner=SSH_USER, group='keystone',
759
perms=0o644)
760
761
762
@retry_on_exception(3, base_delay=2, exc_type=subprocess.CalledProcessError)
763
def unison_sync(paths_to_sync):
764
"""Do unison sync and retry a few times if it fails since peers may not be
765
ready for sync.
766
"""
767
log('Synchronizing CA (%s) to all peers.' % (', '.join(paths_to_sync)),
768
level=INFO)
769
keystone_gid = grp.getgrnam('keystone').gr_gid
770
unison.sync_to_peers(peer_interface='cluster', paths=paths_to_sync,
771
user=SSH_USER, verbose=True, gid=keystone_gid,
772
fatal=True)
773
774
775
def get_ssl_sync_request_units():
776
"""Get list of units that have requested to be synced.
777
778
NOTE: this must be called from cluster relation context.
779
"""
780
units = []
781
for unit in related_units():
782
settings = relation_get(unit=unit) or {}
783
rkeys = settings.keys()
784
key = re.compile("^ssl-sync-required-(.+)")
785
for rkey in rkeys:
786
res = re.search(key, rkey)
787
if res:
788
units.append(res.group(1))
789
790
return units
791
792
793
def is_ssl_cert_master():
794
"""Return True if this unit is ssl cert master."""
795
master = None
796
for rid in relation_ids('cluster'):
797
master = relation_get(attribute='ssl-cert-master', rid=rid,
798
unit=local_unit())
799
800
return master == local_unit()
801
802
803
def ensure_ssl_cert_master(use_oldest_peer=False):
804
"""Ensure that an ssl cert master has been elected.
805
806
Normally the cluster leader will take control but we allow for this to be
807
ignored since this could be called before the cluster is ready.
808
"""
809
# Don't do anything if we are not in ssl/https mode
810
if not (is_str_true(config('use-https')) or
811
is_str_true(config('https-service-endpoints'))):
812
log("SSL/HTTPS is NOT enabled", level=DEBUG)
813
return False
814
815
if not peer_units():
816
log("Not syncing certs since there are no peer units.", level=INFO)
817
return False
818
819
if use_oldest_peer:
820
elect = oldest_peer(peer_units())
821
else:
822
elect = is_elected_leader(CLUSTER_RES)
823
824
if elect:
825
masters = []
826
for rid in relation_ids('cluster'):
827
for unit in related_units(rid):
828
m = relation_get(rid=rid, unit=unit,
829
attribute='ssl-cert-master')
830
if m is not None:
831
masters.append(m)
832
833
# We expect all peers to echo this setting
834
if not masters or 'unknown' in masters:
835
log("Notifying peers this unit is ssl-cert-master", level=INFO)
836
for rid in relation_ids('cluster'):
837
settings = {'ssl-cert-master': local_unit()}
838
relation_set(relation_id=rid, relation_settings=settings)
839
840
# Return now and wait for cluster-relation-changed (peer_echo) for
841
# sync.
842
return False
843
elif len(set(masters)) != 1 and local_unit() not in masters:
844
log("Did not get concensus from peers on who is master (%s) - "
845
"waiting for current master to release before self-electing" %
846
(masters), level=INFO)
847
return False
848
849
if not is_ssl_cert_master():
850
log("Not ssl cert master - skipping sync", level=INFO)
851
return False
852
853
return True
854
855
856
def synchronize_ca(fatal=False):
857
"""Broadcast service credentials to peers.
858
859
By default a failure to sync is fatal and will result in a raised
860
exception.
861
862
This function uses a relation setting 'ssl-cert-master' to get some
863
leader stickiness while synchronisation is being carried out. This ensures
864
that the last host to create and broadcast cetificates has the option to
865
complete actions before electing the new leader as sync master.
866
"""
867
paths_to_sync = [SYNC_FLAGS_DIR]
868
869
if is_str_true(config('https-service-endpoints')):
870
log("Syncing all endpoint certs since https-service-endpoints=True",
871
level=DEBUG)
872
paths_to_sync.append(SSL_DIR)
873
paths_to_sync.append(APACHE_SSL_DIR)
874
paths_to_sync.append(CA_CERT_PATH)
875
elif is_str_true(config('use-https')):
876
log("Syncing keystone-endpoint certs since use-https=True",
877
level=DEBUG)
878
paths_to_sync.append(APACHE_SSL_DIR)
879
paths_to_sync.append(CA_CERT_PATH)
880
881
if not paths_to_sync:
882
log("Nothing to sync - skipping", level=DEBUG)
883
return
884
885
if not os.path.isdir(SYNC_FLAGS_DIR):
886
mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775)
887
888
# We need to restart peer apache services to ensure they have picked up
889
# new ssl keys.
890
create_peer_service_actions('restart', ['apache2'])
891
create_peer_actions(['update-ca-certificates'])
892
893
# Format here needs to match that used when peers request sync
894
synced_units = [unit.replace('/', '-') for unit in peer_units()]
895
896
retries = 3
897
while True:
898
hash1 = hashlib.sha256()
899
for path in paths_to_sync:
900
update_hash_from_path(hash1, path)
901
902
try:
903
unison_sync(paths_to_sync)
904
except:
905
if fatal:
906
raise
907
else:
908
log("Sync failed but fatal=False", level=INFO)
909
return
910
911
hash2 = hashlib.sha256()
912
for path in paths_to_sync:
913
update_hash_from_path(hash2, path)
914
915
# Detect whether someone else has synced to this unit while we did our
916
# transfer.
917
if hash1.hexdigest() != hash2.hexdigest():
918
retries -= 1
919
if retries > 0:
920
log("SSL dir contents changed during sync - retrying unison "
921
"sync %s more times" % (retries), level=WARNING)
922
else:
923
log("SSL dir contents changed during sync - retries failed",
924
level=ERROR)
925
return {}
926
else:
927
break
928
929
hash = hash1.hexdigest()
930
log("Sending restart-services-trigger=%s to all peers" % (hash),
931
level=DEBUG)
932
933
log("Sync complete", level=DEBUG)
934
return {'restart-services-trigger': hash,
935
'ssl-synced-units': json.dumps(synced_units)}
936
937
938
def update_hash_from_path(hash, path, recurse_depth=10):
939
"""Recurse through path and update the provided hash for every file found.
940
"""
941
if not recurse_depth:
942
log("Max recursion depth (%s) reached for update_hash_from_path() at "
943
"path='%s' - not going any deeper" % (recurse_depth, path),
944
level=WARNING)
945
return
946
947
for p in glob.glob("%s/*" % path):
948
if os.path.isdir(p):
949
update_hash_from_path(hash, p, recurse_depth=recurse_depth - 1)
950
else:
951
with open(p, 'r') as fd:
952
hash.update(fd.read())
953
954
955
def synchronize_ca_if_changed(force=False, fatal=False):
956
"""Decorator to perform ssl cert sync if decorated function modifies them
957
in any way.
958
959
If force is True a sync is done regardless.
960
"""
961
def inner_synchronize_ca_if_changed1(f):
962
def inner_synchronize_ca_if_changed2(*args, **kwargs):
963
# Only sync master can do sync. Ensure (a) we are not nested and
964
# (b) a master is elected and we are it.
965
acquired = SSL_SYNC_SEMAPHORE.acquire(blocking=0)
966
try:
967
if not acquired:
968
log("Nested sync - ignoring", level=DEBUG)
969
return f(*args, **kwargs)
970
971
if not ensure_ssl_cert_master():
972
log("Not leader - ignoring sync", level=DEBUG)
973
return f(*args, **kwargs)
974
975
peer_settings = {}
976
if not force:
977
ssl_dirs = [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]
978
979
hash1 = hashlib.sha256()
980
for path in ssl_dirs:
981
update_hash_from_path(hash1, path)
982
983
ret = f(*args, **kwargs)
984
985
hash2 = hashlib.sha256()
986
for path in ssl_dirs:
987
update_hash_from_path(hash2, path)
988
989
if hash1.hexdigest() != hash2.hexdigest():
990
log("SSL certs have changed - syncing peers",
991
level=DEBUG)
992
peer_settings = synchronize_ca(fatal=fatal)
993
else:
994
log("SSL certs have not changed - skipping sync",
995
level=DEBUG)
996
else:
997
ret = f(*args, **kwargs)
998
log("Doing forced ssl cert sync", level=DEBUG)
999
peer_settings = synchronize_ca(fatal=fatal)
1000
1001
# If we are the sync master but not leader, ensure we have
1002
# relinquished master status.
1003
if not is_elected_leader(CLUSTER_RES):
1004
log("Re-electing ssl cert master.", level=INFO)
1005
peer_settings['ssl-cert-master'] = 'unknown'
1006
1007
if peer_settings:
1008
for rid in relation_ids('cluster'):
1009
relation_set(relation_id=rid,
1010
relation_settings=peer_settings)
1011
1012
return ret
1013
finally:
1014
SSL_SYNC_SEMAPHORE.release()
1015
1016
return inner_synchronize_ca_if_changed2
1017
1018
return inner_synchronize_ca_if_changed1
638
1019
639
1020
640
1021
def get_ca(user='keystone', group='keystone'):
642
1023
Initialize a new CA object if one hasn't already been loaded.
643
1024
This will create a new CA or load an existing one.
644
1025
"""
645
if not CA:
1026
if not ssl.CA_SINGLETON:
646
1027
if not os.path.isdir(SSL_DIR):
647
1028
os.mkdir(SSL_DIR)
1029
648
1030
d_name = '_'.join(SSL_CA_NAME.lower().split(' '))
649
1031
ca = ssl.JujuCA(name=SSL_CA_NAME, user=user, group=group,
650
1032
ca_dir=os.path.join(SSL_DIR,
651
1033
'%s_intermediate_ca' % d_name),
652
1034
root_ca_dir=os.path.join(SSL_DIR,
653
1035
'%s_root_ca' % d_name))
1036
654
1037
# SSL_DIR is synchronized via all peers over unison+ssh, need
655
1038
# to ensure permissions.
656
1039
subprocess.check_output(['chown', '-R', '%s.%s' % (user, group),
657
1040
'%s' % SSL_DIR])
658
1041
subprocess.check_output(['chmod', '-R', 'g+rwx', '%s' % SSL_DIR])
659
CA.append(ca)
660
return CA[0]
1042
1043
# Ensure a master has been elected and prefer this unit. Note that we
1044
# prefer oldest peer as predicate since this action i normally only
1045
# performed once at deploy time when the oldest peer should be the
1046
# first to be ready.
1047
ensure_ssl_cert_master(use_oldest_peer=True)
1048
1049
ssl.CA_SINGLETON.append(ca)
1050
1051
return ssl.CA_SINGLETON[0]
661
1052
662
1053
663
1054
def relation_list(rid):
683
1074
https_cns = []
684
1075
if single.issubset(settings):
685
1076
# other end of relation advertised only one endpoint
686
if 'None' in [v for k, v in settings.iteritems()]:
1077
if 'None' in settings.itervalues():
687
1078
# Some backend services advertise no endpoint but require a
688
1079
# hook execution to update auth strategy.
689
1080
relation_data = {}
699
1090
relation_data["auth_port"] = config('admin-port')
700
1091
relation_data["service_port"] = config('service-port')
701
1092
relation_data["region"] = config('region')
702
if config('https-service-endpoints') in ['True', 'true']:
1093
if is_str_true(config('https-service-endpoints')):
703
1094
# Pass CA cert as client will need it to
704
1095
# verify https connections
705
1096
ca = get_ca(user=SSH_USER)
711
1102
for role in get_requested_roles(settings):
712
1103
log("Creating requested role: %s" % role)
713
1104
create_role(role)
1105
714
1106
peer_store_and_set(relation_id=relation_id,
715
1107
**relation_data)
716
1108
return
786
1178
if prefix:
787
1179
service_username = "%s%s" % (prefix, service_username)
788
1180
789
if 'None' in [v for k, v in settings.iteritems()]:
1181
if 'None' in settings.itervalues():
790
1182
return
791
1183
792
1184
if not service_username:
838
1230
relation_data["auth_protocol"] = "http"
839
1231
relation_data["service_protocol"] = "http"
840
1232
# generate or get a new cert/key for service if set to manage certs.
841
if config('https-service-endpoints') in ['True', 'true']:
1233
if is_str_true(config('https-service-endpoints')):
842
1234
ca = get_ca(user=SSH_USER)
843
1235
# NOTE(jamespage) may have multiple cns to deal with to iterate
844
1236
https_cns = set(https_cns)
853
1245
ca_bundle = ca.get_ca_bundle()
854
1246
relation_data['ca_cert'] = b64encode(ca_bundle)
855
1247
relation_data['https_keystone'] = 'True'
1248
856
1249
peer_store_and_set(relation_id=relation_id,
857
1250
**relation_data)
858
1251
Loggerhead is a web-based interface for Breezy
Version: 2.0.1