3
# Copyright (C) 2013 Canonical Ltd.
5
# Author: Scott Moser <scott.moser@canonical.com>
7
# This program is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU General Public License version 3, as
9
# published by the Free Software Foundation.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program. If not, see <http://www.gnu.org/licenses/>.
26
import xml.etree.ElementTree as ET
28
from xml.dom import minidom
30
from cloudinit.sources.helpers.azure import get_metadata_from_fabric
32
from cloudinit import log as logging
33
from cloudinit.settings import PER_ALWAYS
34
from cloudinit import sources
35
from cloudinit import util
37
LOG = logging.getLogger(__name__)
40
DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"}
41
AGENT_START = ['service', 'walinuxagent', 'start']
44
"i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"
48
'agent_command': AGENT_START,
49
'data_dir': "/var/lib/waagent",
54
'command': BOUNCE_COMMAND,
55
'hostname_command': 'hostname',
57
'disk_aliases': {'ephemeral0': '/dev/sdb'},
60
BUILTIN_CLOUD_CONFIG = {
62
'ephemeral0': {'table_type': 'gpt',
66
'fs_setup': [{'filesystem': 'ext4',
67
'device': 'ephemeral0.1',
68
'replace_fs': 'ntfs'}],
71
DS_CFG_PATH = ['datasource', DS_NAME]
72
DEF_EPHEMERAL_LABEL = 'Temporary Storage'
74
# The redacted password fails to meet password complexity requirements
75
# so we can safely use this to mask/redact the password in the ovf-env.xml
76
DEF_PASSWD_REDACTION = 'REDACTED'
79
def get_hostname(hostname_command='hostname'):
80
return util.subp(hostname_command, capture=True)[0].strip()
83
def set_hostname(hostname, hostname_command='hostname'):
84
util.subp([hostname_command, hostname])
87
@contextlib.contextmanager
88
def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'):
90
Set a temporary hostname, restoring the previous hostname on exit.
92
Will have the value of the previous hostname when used as a context
93
manager, or None if the hostname was not changed.
95
policy = cfg['hostname_bounce']['policy']
96
previous_hostname = get_hostname(hostname_command)
97
if (not util.is_true(cfg.get('set_hostname')) or
98
util.is_false(policy) or
99
(previous_hostname == temp_hostname and policy != 'force')):
102
set_hostname(temp_hostname, hostname_command)
104
yield previous_hostname
106
set_hostname(previous_hostname, hostname_command)
109
class DataSourceAzureNet(sources.DataSource):
110
def __init__(self, sys_cfg, distro, paths):
111
sources.DataSource.__init__(self, sys_cfg, distro, paths)
112
self.seed_dir = os.path.join(paths.seed_dir, 'azure')
115
self.ds_cfg = util.mergemanydict([
116
util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
120
root = sources.DataSource.__str__(self)
121
return "%s [seed=%s]" % (root, self.seed)
123
def get_metadata_from_agent(self):
124
temp_hostname = self.metadata.get('local-hostname')
125
hostname_command = self.ds_cfg['hostname_bounce']['hostname_command']
126
with temporary_hostname(temp_hostname, self.ds_cfg,
127
hostname_command=hostname_command) \
128
as previous_hostname:
129
if (previous_hostname is not None and
130
util.is_true(self.ds_cfg.get('set_hostname'))):
131
cfg = self.ds_cfg['hostname_bounce']
133
perform_hostname_bounce(hostname=temp_hostname,
135
prev_hostname=previous_hostname)
136
except Exception as e:
137
LOG.warn("Failed publishing hostname: %s", e)
138
util.logexc(LOG, "handling set_hostname failed")
141
invoke_agent(self.ds_cfg['agent_command'])
142
except util.ProcessExecutionError:
143
# claim the datasource even if the command failed
144
util.logexc(LOG, "agent command '%s' failed.",
145
self.ds_cfg['agent_command'])
147
ddir = self.ds_cfg['data_dir']
151
for pk in self.cfg.get('_pubkeys', []):
152
if pk.get('value', None):
153
key_value = pk['value']
154
LOG.debug("ssh authentication: using value from fabric")
156
bname = str(pk['fingerprint'] + ".crt")
157
fp_files += [os.path.join(ddir, bname)]
158
LOG.debug("ssh authentication: "
159
"using fingerprint from fabirc")
161
missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
165
LOG.warn("Did not find files, but going on: %s", missing)
168
metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
172
# azure removes/ejects the cdrom containing the ovf-env.xml
173
# file on reboot. So, in order to successfully reboot we
174
# need to look in the datadir and consider that valid
175
ddir = self.ds_cfg['data_dir']
177
candidates = [self.seed_dir]
178
candidates.extend(list_possible_azure_ds_devs())
180
candidates.append(ddir)
184
for cdev in candidates:
186
if cdev.startswith("/dev/"):
187
ret = util.mount_cb(cdev, load_azure_ds_dir)
189
ret = load_azure_ds_dir(cdev)
191
except NonAzureDataSource:
193
except BrokenAzureDataSource as exc:
195
except util.MountFailedError:
196
LOG.warn("%s was not mountable", cdev)
199
(md, self.userdata_raw, cfg, files) = ret
201
self.metadata = util.mergemanydict([md, DEFAULT_METADATA])
202
self.cfg = util.mergemanydict([cfg, BUILTIN_CLOUD_CONFIG])
205
LOG.debug("found datasource in %s", cdev)
212
LOG.debug("using files cached in %s", ddir)
214
# azure / hyper-v provides random data here
215
seed = util.load_file("/sys/firmware/acpi/tables/OEM0",
216
quiet=True, decode=False)
218
self.metadata['random_seed'] = seed
220
# now update ds_cfg to reflect contents pass in config
221
user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
222
self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
224
# walinux agent writes files world readable, but expects
225
# the directory to be protected.
226
write_files(ddir, files, dirmode=0o700)
228
if self.ds_cfg['agent_command'] == '__builtin__':
229
metadata_func = get_metadata_from_fabric
231
metadata_func = self.get_metadata_from_agent
233
fabric_data = metadata_func()
234
except Exception as exc:
235
LOG.info("Error communicating with Azure fabric; assume we aren't"
236
" on Azure.", exc_info=True)
239
self.metadata['instance-id'] = util.read_dmi_data('system-uuid')
240
self.metadata.update(fabric_data)
242
found_ephemeral = find_fabric_formatted_ephemeral_disk()
244
self.ds_cfg['disk_aliases']['ephemeral0'] = found_ephemeral
245
LOG.debug("using detected ephemeral0 of %s", found_ephemeral)
247
cc_modules_override = support_new_ephemeral(self.sys_cfg)
248
if cc_modules_override:
249
self.cfg['cloud_config_modules'] = cc_modules_override
253
def device_name_to_device(self, name):
254
return self.ds_cfg['disk_aliases'].get(name)
256
def get_config_obj(self):
259
def check_instance_id(self, sys_cfg):
260
# quickly (local check only) if self.instance_id is still valid
261
return sources.instance_id_matches_system_uuid(self.get_instance_id())
265
return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*'))
268
def find_fabric_formatted_ephemeral_part():
270
Locate the first fabric formatted ephemeral device.
272
potential_locations = ['/dev/disk/cloud/azure_resource-part1',
273
'/dev/disk/azure/resource-part1']
274
device_location = None
275
for potential_location in potential_locations:
276
if os.path.exists(potential_location):
277
device_location = potential_location
279
if device_location is None:
281
ntfs_devices = util.find_devs_with("TYPE=ntfs")
282
real_device = os.path.realpath(device_location)
283
if real_device in ntfs_devices:
284
return device_location
288
def find_fabric_formatted_ephemeral_disk():
290
Get the ephemeral disk.
292
part_dev = find_fabric_formatted_ephemeral_part()
294
return part_dev.split('-')[0]
298
def support_new_ephemeral(cfg):
300
Windows Azure makes ephemeral devices ephemeral to boot; a ephemeral device
301
may be presented as a fresh device, or not.
303
Since the knowledge of when a disk is supposed to be plowed under is
304
specific to Windows Azure, the logic resides here in the datasource. When a
305
new ephemeral device is detected, cloud-init overrides the default
306
frequency for both disk-setup and mounts for the current boot only.
308
device = find_fabric_formatted_ephemeral_part()
310
LOG.debug("no default fabric formated ephemeral0.1 found")
312
LOG.debug("fabric formated ephemeral0.1 device at %s", device)
316
file_count = util.mount_cb(device, count_files)
319
LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count)
322
LOG.debug("fabric prepared ephemeral0.1 will be preserved")
325
# if device was already mounted, then we need to unmount it
326
# race conditions could allow for a check-then-unmount
327
# to have a false positive. so just unmount and then check.
329
util.subp(['umount', device])
330
except util.ProcessExecutionError as e:
331
if device in util.mounts():
332
LOG.warn("Failed to unmount %s, will not reformat.", device)
333
LOG.debug("Failed umount: %s", e)
336
LOG.debug("cloud-init will format ephemeral0.1 this boot.")
337
LOG.debug("setting disk_setup and mounts modules 'always' for this boot")
339
cc_modules = cfg.get('cloud_config_modules')
344
for mod in cc_modules:
345
if mod in ("disk_setup", "mounts"):
346
mod_list.append([mod, PER_ALWAYS])
347
LOG.debug("set module '%s' to 'always' for this boot", mod)
353
def perform_hostname_bounce(hostname, cfg, prev_hostname):
354
# set the hostname to 'hostname' if it is not already set to that.
355
# then, if policy is not off, bounce the interface using command
356
command = cfg['command']
357
interface = cfg['interface']
358
policy = cfg['policy']
360
msg = ("hostname=%s policy=%s interface=%s" %
361
(hostname, policy, interface))
362
env = os.environ.copy()
363
env['interface'] = interface
364
env['hostname'] = hostname
365
env['old_hostname'] = prev_hostname
367
if command == "builtin":
368
command = BOUNCE_COMMAND
370
LOG.debug("pubhname: publishing hostname [%s]", msg)
371
shell = not isinstance(command, (list, tuple))
372
# capture=False, see comments in bug 1202758 and bug 1206164.
373
util.log_time(logfunc=LOG.debug, msg="publishing hostname",
374
get_uptime=True, func=util.subp,
375
kwargs={'args': command, 'shell': shell, 'capture': False,
379
def crtfile_to_pubkey(fname, data=None):
380
pipeline = ('openssl x509 -noout -pubkey < "$0" |'
381
'ssh-keygen -i -m PKCS8 -f /dev/stdin')
382
(out, _err) = util.subp(['sh', '-c', pipeline, fname],
383
capture=True, data=data)
387
def pubkeys_from_crt_files(flist):
392
pubkeys.append(crtfile_to_pubkey(fname))
393
except util.ProcessExecutionError:
397
LOG.warn("failed to convert the crt files to pubkey: %s", errors)
402
def wait_for_files(flist, maxwait=60, naplen=.5):
405
while waited < maxwait:
406
need -= set([f for f in need if os.path.exists(f)])
414
def write_files(datadir, files, dirmode=None):
416
def _redact_password(cnt, fname):
417
"""Azure provides the UserPassword in plain text. So we redact it"""
419
root = ET.fromstring(cnt)
420
for elem in root.iter():
421
if ('UserPassword' in elem.tag and
422
elem.text != DEF_PASSWD_REDACTION):
423
elem.text = DEF_PASSWD_REDACTION
424
return ET.tostring(root)
426
LOG.critical("failed to redact userpassword in %s", fname)
433
util.ensure_dir(datadir, dirmode)
434
for (name, content) in files.items():
435
fname = os.path.join(datadir, name)
436
if 'ovf-env.xml' in name:
437
content = _redact_password(content, fname)
438
util.write_file(filename=fname, content=content, mode=0o600)
441
def invoke_agent(cmd):
442
# this is a function itself to simplify patching it for test
444
LOG.debug("invoking agent: %s", cmd)
445
util.subp(cmd, shell=(not isinstance(cmd, list)))
447
LOG.debug("not invoking agent")
450
def find_child(node, filter_func):
452
if not node.hasChildNodes():
454
for child in node.childNodes:
455
if filter_func(child):
460
def load_azure_ovf_pubkeys(sshnode):
461
# This parses a 'SSH' node formatted like below, and returns
463
# [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7',
464
# 'path': 'where/to/go'}]
467
# <PublicKey><Fingerprint>ABC</FingerPrint><Path>/ABC</Path>
469
# </PublicKeys></SSH>
470
results = find_child(sshnode, lambda n: n.localName == "PublicKeys")
471
if len(results) == 0:
474
raise BrokenAzureDataSource("Multiple 'PublicKeys'(%s) in SSH node" %
477
pubkeys_node = results[0]
478
pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey")
480
if len(pubkeys) == 0:
484
text_node = minidom.Document.TEXT_NODE
486
for pk_node in pubkeys:
487
if not pk_node.hasChildNodes():
490
cur = {'fingerprint': "", 'path': "", 'value': ""}
491
for child in pk_node.childNodes:
492
if child.nodeType == text_node or not child.localName:
495
name = child.localName.lower()
497
if name not in cur.keys():
500
if (len(child.childNodes) != 1 or
501
child.childNodes[0].nodeType != text_node):
504
cur[name] = child.childNodes[0].wholeText.strip()
510
def read_azure_ovf(contents):
512
dom = minidom.parseString(contents)
513
except Exception as e:
514
raise BrokenAzureDataSource("invalid xml: %s" % e)
516
results = find_child(dom.documentElement,
517
lambda n: n.localName == "ProvisioningSection")
519
if len(results) == 0:
520
raise NonAzureDataSource("No ProvisioningSection")
522
raise BrokenAzureDataSource("found '%d' ProvisioningSection items" %
524
provSection = results[0]
526
lpcs_nodes = find_child(provSection,
528
n.localName == "LinuxProvisioningConfigurationSet")
530
if len(results) == 0:
531
raise NonAzureDataSource("No LinuxProvisioningConfigurationSet")
533
raise BrokenAzureDataSource("found '%d' %ss" %
534
("LinuxProvisioningConfigurationSet",
538
if not lpcs.hasChildNodes():
539
raise BrokenAzureDataSource("no child nodes of configuration set")
541
md_props = 'seedfrom'
542
md = {'azure_data': {}}
548
for child in lpcs.childNodes:
549
if child.nodeType == dom.TEXT_NODE or not child.localName:
552
name = child.localName.lower()
556
if (len(child.childNodes) == 1 and
557
child.childNodes[0].nodeType == dom.TEXT_NODE):
559
value = child.childNodes[0].wholeText
561
attrs = dict([(k, v) for k, v in child.attributes.items()])
563
# we accept either UserData or CustomData. If both are present
564
# then behavior is undefined.
565
if name == "userdata" or name == "customdata":
566
if attrs.get('encoding') in (None, "base64"):
567
ud = base64.b64decode(''.join(value.split()))
570
elif name == "username":
572
elif name == "userpassword":
574
elif name == "hostname":
575
md['local-hostname'] = value
576
elif name == "dscfg":
577
if attrs.get('encoding') in (None, "base64"):
578
dscfg = base64.b64decode(''.join(value.split()))
581
cfg['datasource'] = {DS_NAME: util.load_yaml(dscfg, default={})}
583
cfg['_pubkeys'] = load_azure_ovf_pubkeys(child)
584
elif name == "disablesshpasswordauthentication":
585
cfg['ssh_pwauth'] = util.is_false(value)
590
md['azure_data'][name] = value
594
defuser['name'] = username
595
if password and DEF_PASSWD_REDACTION != password:
596
defuser['passwd'] = encrypt_pass(password)
597
defuser['lock_passwd'] = False
600
cfg['system_info'] = {'default_user': defuser}
602
if 'ssh_pwauth' not in cfg and password:
603
cfg['ssh_pwauth'] = True
608
def encrypt_pass(password, salt_id="$6$"):
609
return crypt.crypt(password, salt_id + util.rand_str(strlen=16))
612
def list_possible_azure_ds_devs():
613
# return a sorted list of devices that might have a azure datasource
615
for fstype in ("iso9660", "udf"):
616
devlist.extend(util.find_devs_with("TYPE=%s" % fstype))
618
devlist.sort(reverse=True)
622
def load_azure_ds_dir(source_dir):
623
ovf_file = os.path.join(source_dir, "ovf-env.xml")
625
if not os.path.isfile(ovf_file):
626
raise NonAzureDataSource("No ovf-env file found")
628
with open(ovf_file, "rb") as fp:
631
md, ud, cfg = read_azure_ovf(contents)
632
return (md, ud, cfg, {'ovf-env.xml': contents})
635
class BrokenAzureDataSource(Exception):
639
class NonAzureDataSource(Exception):
643
# Used to match classes to dependencies
645
(DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
649
# Return a list of data sources that match this set of dependencies
650
def get_datasource_list(depends):
651
return sources.list_from_depends(depends, datasources)
added
removed
cloudinit/sources/DataSourceAzure.py
