3
# Copyright (C) 2012 Canonical Ltd.
4
# Copyright (C) 2012 Yahoo! Inc.
6
# Author: Scott Moser <scott.moser@canonical.com>
7
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
9
# This program is free software: you can redistribute it and/or modify
10
# it under the terms of the GNU General Public License version 3, as
11
# published by the Free Software Foundation.
13
# This program is distributed in the hope that it will be useful,
14
# but WITHOUT ANY WARRANTY; without even the implied warranty of
15
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
# GNU General Public License for more details.
18
# You should have received a copy of the GNU General Public License
19
# along with this program. If not, see <http://www.gnu.org/licenses/>.
29
from cloudinit import ec2_utils
30
from cloudinit import log as logging
31
from cloudinit import net
32
from cloudinit import sources
33
from cloudinit import url_helper
34
from cloudinit import util
36
# For reference: http://tinyurl.com/laora4c
38
LOG = logging.getLogger(__name__)
41
# Path <-> (metadata key name, translator function, default value)
42
'etc/network/interfaces': ('network_config', lambda x: x, ''),
43
'meta.js': ('meta_js', util.load_json, {}),
44
"root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''),
47
# Cloud-init metadata names <-> (metadata key, is required)
48
('local-hostname', 'hostname', False),
49
('instance-id', 'uuid', True),
52
OS_FOLSOM = '2012-08-10'
53
OS_GRIZZLY = '2013-04-04'
54
OS_HAVANA = '2013-10-17'
55
OS_LIBERTY = '2015-10-15'
56
# keep this in chronological order. new supported versions go at the end.
65
class NonReadable(IOError):
69
class BrokenMetadata(IOError):
73
class SourceMixin(object):
74
def _ec2_name_to_device(self, name):
75
if not self.ec2_metadata:
77
bdm = self.ec2_metadata.get('block-device-mapping', {})
78
for (ent_name, device) in bdm.items():
83
def get_public_ssh_keys(self):
87
return sources.normalize_pubkey_data(self.metadata.get(name))
89
def _os_name_to_device(self, name):
92
criteria = 'LABEL=%s' % (name)
94
criteria = 'TYPE=%s' % (name)
95
dev_entries = util.find_devs_with(criteria)
97
device = dev_entries[0]
98
except util.ProcessExecutionError:
102
def _validate_device_name(self, device):
105
if not device.startswith("/"):
106
device = "/dev/%s" % device
107
if os.path.exists(device):
109
# Durn, try adjusting the mapping
110
remapped = self._remap_device(os.path.basename(device))
112
LOG.debug("Remapped device name %s => %s", device, remapped)
116
def device_name_to_device(self, name):
117
# Translate a 'name' to a 'physical' device
120
# Try the ec2 mapping first
123
names.insert(0, 'ami')
127
LOG.debug("Using ec2 style lookup to find device %s", names)
129
device = self._ec2_name_to_device(n)
130
device = self._validate_device_name(device)
133
# Try the openstack way second
135
LOG.debug("Using openstack style lookup to find device %s", names)
137
device = self._os_name_to_device(n)
138
device = self._validate_device_name(device)
145
LOG.debug("Mapped %s to device %s", name, device)
149
@six.add_metaclass(abc.ABCMeta)
150
class BaseReader(object):
152
def __init__(self, base_path):
153
self.base_path = base_path
156
def _path_join(self, base, *add_ons):
160
def _path_read(self, path, decode=False):
164
def _fetch_available_versions(self):
168
def _read_ec2_metadata(self):
171
def _find_working_version(self):
173
versions_available = self._fetch_available_versions()
174
except Exception as e:
175
LOG.debug("Unable to read openstack versions from %s due to: %s",
177
versions_available = []
179
# openstack.OS_VERSIONS is stored in chronological order, so
180
# reverse it to check newest first.
181
supported = [v for v in reversed(list(OS_VERSIONS))]
182
selected_version = OS_LATEST
184
for potential_version in supported:
185
if potential_version not in versions_available:
187
selected_version = potential_version
190
LOG.debug("Selected version '%s' from %s", selected_version,
192
return selected_version
194
def _read_content_path(self, item, decode=False):
195
path = item.get('content_path', '').lstrip("/")
196
path_pieces = path.split("/")
197
valid_pieces = [p for p in path_pieces if len(p)]
199
raise BrokenMetadata("Item %s has no valid content path" % (item))
200
path = self._path_join(self.base_path, "openstack", *path_pieces)
201
return self._path_read(path, decode=decode)
204
"""Reads a version 2 formatted location.
206
Return a dict with metadata, userdata, ec2-metadata, dsmode,
207
network_config, files and version (2).
209
If not a valid location, raise a NonReadable exception.
212
load_json_anytype = functools.partial(
213
util.load_json, root_types=(dict, list) + six.string_types)
215
def datafiles(version):
217
files['metadata'] = (
219
self._path_join("openstack", version, 'meta_data.json'),
222
# Translator function (applied after loading)
225
files['userdata'] = (
226
self._path_join("openstack", version, 'user_data'),
230
files['vendordata'] = (
231
self._path_join("openstack", version, 'vendor_data.json'),
235
files['networkdata'] = (
236
self._path_join("openstack", version, 'network_data.json'),
246
data = datafiles(self._find_working_version())
247
for (name, (path, required, translator)) in data.items():
248
path = self._path_join(self.base_path, path)
252
data = self._path_read(path)
255
LOG.debug("Failed reading optional path %s due"
258
LOG.debug("Failed reading mandatory path %s due"
262
if required and not found:
263
raise NonReadable("Missing mandatory path: %s" % path)
264
if found and translator:
266
data = translator(data)
267
except Exception as e:
268
raise BrokenMetadata("Failed to process "
269
"path %s: %s" % (path, e))
273
metadata = results['metadata']
274
if 'random_seed' in metadata:
275
random_seed = metadata['random_seed']
277
metadata['random_seed'] = base64.b64decode(random_seed)
278
except (ValueError, TypeError) as e:
279
raise BrokenMetadata("Badly formatted metadata"
280
" random_seed entry: %s" % e)
282
# load any files that were provided
284
metadata_files = metadata.get('files', [])
285
for item in metadata_files:
286
if 'path' not in item:
290
files[path] = self._read_content_path(item)
291
except Exception as e:
292
raise BrokenMetadata("Failed to read provided "
293
"file %s: %s" % (path, e))
294
results['files'] = files
296
# The 'network_config' item in metadata is a content pointer
297
# to the network config that should be applied. It is just a
298
# ubuntu/debian '/etc/network/interfaces' file.
299
net_item = metadata.get("network_config", None)
302
content = self._read_content_path(net_item, decode=True)
303
results['network_config'] = content
305
raise BrokenMetadata("Failed to read network"
306
" configuration: %s" % (e))
308
# To openstack, user can specify meta ('nova boot --meta=key=value')
309
# and those will appear under metadata['meta'].
310
# if they specify 'dsmode' they're indicating the mode that they intend
311
# for this datasource to operate in.
313
results['dsmode'] = metadata['meta']['dsmode']
317
# Read any ec2-metadata (if applicable)
318
results['ec2-metadata'] = self._read_ec2_metadata()
320
# Perform some misc. metadata key renames...
321
for (target_key, source_key, is_required) in KEY_COPIES:
322
if is_required and source_key not in metadata:
323
raise BrokenMetadata("No '%s' entry in metadata" % source_key)
324
if source_key in metadata:
325
metadata[target_key] = metadata.get(source_key)
329
class ConfigDriveReader(BaseReader):
330
def __init__(self, base_path):
331
super(ConfigDriveReader, self).__init__(base_path)
332
self._versions = None
334
def _path_join(self, base, *add_ons):
335
components = [base] + list(add_ons)
336
return os.path.join(*components)
338
def _path_read(self, path, decode=False):
339
return util.load_file(path, decode=decode)
341
def _fetch_available_versions(self):
342
if self._versions is None:
343
path = self._path_join(self.base_path, 'openstack')
344
found = [d for d in os.listdir(path)
345
if os.path.isdir(os.path.join(path))]
346
self._versions = sorted(found)
347
return self._versions
349
def _read_ec2_metadata(self):
350
path = self._path_join(self.base_path,
351
'ec2', 'latest', 'meta-data.json')
352
if not os.path.exists(path):
356
return util.load_json(self._path_read(path))
357
except Exception as e:
358
raise BrokenMetadata("Failed to process "
359
"path %s: %s" % (path, e))
362
"""Reads a version 1 formatted location.
364
Return a dict with metadata, userdata, dsmode, files and version (1).
366
If not a valid path, raise a NonReadable exception.
370
for name in FILES_V1.keys():
371
path = self._path_join(self.base_path, name)
372
if os.path.exists(path):
375
raise NonReadable("%s: no files found" % (self.base_path))
378
for (name, (key, translator, default)) in FILES_V1.items():
382
contents = self._path_read(path)
384
raise BrokenMetadata("Failed to read: %s" % path)
386
md[key] = translator(contents)
387
except Exception as e:
388
raise BrokenMetadata("Failed to process "
389
"path %s: %s" % (path, e))
391
md[key] = copy.deepcopy(default)
393
keydata = md['authorized_keys']
394
meta_js = md['meta_js']
396
# keydata in meta_js is preferred over "injected"
397
keydata = meta_js.get('public-keys', keydata)
399
lines = keydata.splitlines()
400
md['public-keys'] = [l for l in lines
401
if len(l) and not l.startswith("#")]
403
# config-drive-v1 has no way for openstack to provide the instance-id
404
# so we copy that into metadata from the user input
405
if 'instance-id' in meta_js:
406
md['instance-id'] = meta_js['instance-id']
413
# allow the user to specify 'dsmode' in a meta tag
414
if 'dsmode' in meta_js:
415
results['dsmode'] = meta_js['dsmode']
417
# config-drive-v1 has no way of specifying user-data, so the user has
418
# to cheat and stuff it in a meta tag also.
419
results['userdata'] = meta_js.get('user-data', '')
421
# this implementation does not support files other than
422
# network/interfaces and authorized_keys...
423
results['files'] = {}
428
class MetadataReader(BaseReader):
429
def __init__(self, base_url, ssl_details=None, timeout=5, retries=5):
430
super(MetadataReader, self).__init__(base_url)
431
self.ssl_details = ssl_details
432
self.timeout = float(timeout)
433
self.retries = int(retries)
434
self._versions = None
436
def _fetch_available_versions(self):
437
# <baseurl>/openstack/ returns a newline separated list of versions
438
if self._versions is not None:
439
return self._versions
441
version_path = self._path_join(self.base_path, "openstack")
442
content = self._path_read(version_path)
443
for line in content.splitlines():
448
self._versions = found
449
return self._versions
451
def _path_read(self, path, decode=False):
453
def should_retry_cb(_request_args, cause):
455
code = int(cause.code)
458
except (TypeError, ValueError):
459
# Older versions of requests didn't have a code.
463
response = url_helper.readurl(path,
464
retries=self.retries,
465
ssl_details=self.ssl_details,
466
timeout=self.timeout,
467
exception_cb=should_retry_cb)
469
return response.contents.decode()
471
return response.contents
473
def _path_join(self, base, *add_ons):
474
return url_helper.combine_url(base, *add_ons)
476
def _read_ec2_metadata(self):
477
return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details,
478
timeout=self.timeout,
479
retries=self.retries)
482
# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
483
def convert_net_json(network_json=None, known_macs=None):
484
"""Return a dictionary of network_config by parsing provided
485
OpenStack ConfigDrive NetworkData json format
487
OpenStack network_data.json provides a 3 element dictionary
488
- "links" (links are network devices, physical or virtual)
489
- "networks" (networks are ip network configurations for one or more
491
- services (non-ip services, like dns)
493
networks and links are combined via network items referencing specific
494
links via a 'link_id' which maps to a links 'id' field.
496
To convert this format to network_config yaml, we first iterate over the
497
links and then walk the network list to determine if any of the networks
498
utilize the current link; if so we generate a subnet entry for the device
500
We also need to map network_data.json fields to network_config fields. For
501
example, the network_data links 'id' field is equivalent to network_config
502
'name' field for devices. We apply more of this mapping to the various
503
link types that we encounter.
505
There are additional fields that are populated in the network_data.json
506
from OpenStack that are not relevant to network_config yaml, so we
507
enumerate a dictionary of valid keys for network_yaml and apply filtering
508
to drop these superflous keys from the network_config yaml.
510
if network_json is None:
513
# dict of network_config key for filtering network_json
538
links = network_json.get('links', [])
539
networks = network_json.get('networks', [])
540
services = network_json.get('services', [])
545
cfg = dict((k, v) for k, v in link.items()
546
if k in valid_keys['physical'])
547
# 'name' is not in openstack spec yet, but we will support it if it is
548
# present. The 'id' in the spec is currently implemented as the host
549
# nic's name, meaning something like 'tap-adfasdffd'. We do not want
550
# to name guest devices with such ugly names.
552
cfg['name'] = link['name']
554
for network in [n for n in networks
555
if n['link'] == link['id']]:
556
subnet = dict((k, v) for k, v in network.items()
557
if k in valid_keys['subnet'])
558
if 'dhcp' in network['type']:
559
t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
566
'address': network.get('ip_address'),
568
if network['type'] == 'ipv4':
569
subnet['ipv4'] = True
570
if network['type'] == 'ipv6':
571
subnet['ipv6'] = True
572
subnets.append(subnet)
573
cfg.update({'subnets': subnets})
574
if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']:
577
'mac_address': link['ethernet_mac_address']})
578
elif link['type'] in ['bond']:
580
for k, v in link.items():
581
if k == 'bond_links':
583
elif k.startswith('bond'):
584
params.update({k: v})
586
'bond_interfaces': copy.deepcopy(link['bond_links']),
589
elif link['type'] in ['vlan']:
591
'name': "%s.%s" % (link['vlan_link'],
593
'vlan_link': link['vlan_link'],
594
'vlan_id': link['vlan_id'],
595
'mac_address': link['vlan_mac_address'],
599
'Unknown network_data link type: %s' % link['type'])
603
need_names = [d for d in config
604
if d.get('type') == 'physical' and 'name' not in d]
607
if known_macs is None:
608
known_macs = net.get_interfaces_by_mac()
611
mac = d.get('mac_address')
613
raise ValueError("No mac_address or name entry for %s" % d)
614
if mac not in known_macs:
615
raise ValueError("Unable to find a system nic for %s" % d)
616
d['name'] = known_macs[mac]
618
for service in services:
620
cfg.update({'type': 'nameserver'})
623
return {'version': 1, 'config': config}
626
def convert_vendordata_json(data, recurse=True):
627
"""data: a loaded json *object* (strings, arrays, dicts).
628
return something suitable for cloudinit vendordata_raw.
632
string: return string
634
the list is then processed in UserDataProcessor
635
dict: return convert_vendordata_json(data.get('cloud-init'))
639
if isinstance(data, six.string_types):
641
if isinstance(data, list):
642
return copy.deepcopy(data)
643
if isinstance(data, dict):
645
return convert_vendordata_json(data.get('cloud-init'),
647
raise ValueError("vendordata['cloud-init'] cannot be dict")
648
raise ValueError("Unknown data type for vendordata: %s" % type(data))
added
removed
cloudinit/sources/helpers/openstack.py
