3
# Copyright (C) 2009-2010 Canonical Ltd.
4
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
6
# Author: Ben Howard <ben.howard@canonical.com>
8
# This program is free software: you can redistribute it and/or modify
9
# it under the terms of the GNU General Public License version 3, as
10
# published by the Free Software Foundation.
12
# This program is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
17
# You should have received a copy of the GNU General Public License
18
# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
from cloudinit.settings import PER_INSTANCE
20
from cloudinit import util
25
frequency = PER_INSTANCE
27
# Define the commands to use
28
UDEVADM_CMD = util.which('udevadm')
29
SFDISK_CMD = util.which("sfdisk")
30
SGDISK_CMD = util.which("sgdisk")
31
LSBLK_CMD = util.which("lsblk")
32
BLKID_CMD = util.which("blkid")
33
BLKDEV_CMD = util.which("blockdev")
34
WIPEFS_CMD = util.which("wipefs")
36
LOG = logging.getLogger(__name__)
39
def handle(_name, cfg, cloud, log, _args):
41
See doc/examples/cloud-config_disk-setup.txt for documentation on the
44
disk_setup = cfg.get("disk_setup")
45
if isinstance(disk_setup, dict):
46
update_disk_setup_devices(disk_setup, cloud.device_name_to_device)
47
log.debug("Partitioning disks: %s", str(disk_setup))
48
for disk, definition in disk_setup.items():
49
if not isinstance(definition, dict):
50
log.warn("Invalid disk definition for %s" % disk)
54
log.debug("Creating new partition table/disk")
55
util.log_time(logfunc=LOG.debug,
56
msg="Creating partition on %s" % disk,
57
func=mkpart, args=(disk, definition))
58
except Exception as e:
59
util.logexc(LOG, "Failed partitioning operation\n%s" % e)
61
fs_setup = cfg.get("fs_setup")
62
if isinstance(fs_setup, list):
63
log.debug("setting up filesystems: %s", str(fs_setup))
64
update_fs_setup_devices(fs_setup, cloud.device_name_to_device)
65
for definition in fs_setup:
66
if not isinstance(definition, dict):
67
log.warn("Invalid file system definition: %s" % definition)
71
log.debug("Creating new filesystem.")
72
device = definition.get('device')
73
util.log_time(logfunc=LOG.debug,
74
msg="Creating fs for %s" % device,
75
func=mkfs, args=(definition,))
76
except Exception as e:
77
util.logexc(LOG, "Failed during filesystem operation\n%s" % e)
80
def update_disk_setup_devices(disk_setup, tformer):
81
# update 'disk_setup' dictionary anywhere were a device may occur
82
# update it with the response from 'tformer'
83
for origname in disk_setup.keys():
84
transformed = tformer(origname)
85
if transformed is None or transformed == origname:
87
if transformed in disk_setup:
88
LOG.info("Replacing %s in disk_setup for translation of %s",
89
origname, transformed)
90
del disk_setup[transformed]
92
disk_setup[transformed] = disk_setup[origname]
93
disk_setup[transformed]['_origname'] = origname
94
del disk_setup[origname]
95
LOG.debug("updated disk_setup device entry '%s' to '%s'",
96
origname, transformed)
99
def update_fs_setup_devices(disk_setup, tformer):
100
# update 'fs_setup' dictionary anywhere were a device may occur
101
# update it with the response from 'tformer'
102
for definition in disk_setup:
103
if not isinstance(definition, dict):
104
LOG.warn("entry in disk_setup not a dict: %s", definition)
107
origname = definition.get('device')
112
(dev, part) = util.expand_dotted_devname(origname)
114
tformed = tformer(dev)
115
if tformed is not None:
117
LOG.debug("%s is mapped to disk=%s part=%s",
118
origname, tformed, part)
119
definition['_origname'] = origname
120
definition['device'] = tformed
122
if part and 'partition' in definition:
123
definition['_partition'] = definition['partition']
124
definition['partition'] = part
127
def value_splitter(values, start=None):
129
Returns the key/value pairs of output sent as string
130
like: FOO='BAR' HOME='127.0.0.1'
132
_values = shlex.split(values)
134
_values = _values[start:]
136
for key, value in [x.split('=') for x in _values]:
140
def enumerate_disk(device, nodeps=False):
142
Enumerate the elements of a child device.
145
device: the kernel device name
146
nodeps <BOOL>: don't enumerate children devices
148
Return a dict describing the disk:
149
type: the entry type, i.e disk or part
150
fstype: the filesystem type, if it exists
151
label: file system label, if it exists
152
name: the device name, i.e. sda
155
lsblk_cmd = [LSBLK_CMD, '--pairs', '--output', 'NAME,TYPE,FSTYPE,LABEL',
159
lsblk_cmd.append('--nodeps')
163
info, _err = util.subp(lsblk_cmd)
164
except Exception as e:
165
raise Exception("Failed during disk check for %s\n%s" % (device, e))
167
parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
177
for key, value in value_splitter(part):
178
d[key.lower()] = value
183
def device_type(device):
185
Return the device type of the device by calling lsblk.
188
for d in enumerate_disk(device, nodeps=True):
190
return d["type"].lower()
194
def is_device_valid(name, partition=False):
196
Check if the device is a valid device.
200
d_type = device_type(name)
202
LOG.warn("Query against device %s failed" % name)
205
if partition and d_type == 'part':
207
elif not partition and d_type == 'disk':
212
def check_fs(device):
214
Check if the device has a filesystem on it
216
Output of blkid is generally something like:
217
/dev/sda: LABEL="Backup500G" UUID="..." TYPE="ext4"
219
Return values are device, label, type, uuid
221
out, label, fs_type, uuid = None, None, None, None
223
blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device]
225
out, _err = util.subp(blkid_cmd, rcs=[0, 2])
226
except Exception as e:
227
raise Exception("Failed during disk check for %s\n%s" % (device, e))
230
if len(out.splitlines()) == 1:
231
for key, value in value_splitter(out, start=1):
232
if key.lower() == 'label':
234
elif key.lower() == 'type':
236
elif key.lower() == 'uuid':
239
return label, fs_type, uuid
242
def is_filesystem(device):
244
Returns true if the device has a file system.
246
_, fs_type, _ = check_fs(device)
250
def find_device_node(device, fs_type=None, label=None, valid_targets=None,
251
label_match=True, replace_fs=None):
253
Find a device that is either matches the spec, or the first
255
The return is value is (<device>, <bool>) where the device is the
256
device to use and the bool is whether the device matches the
259
Note: This works with GPT partition tables!
261
# label of None is same as no label
265
if not valid_targets:
266
valid_targets = ['disk', 'part']
268
raw_device_used = False
269
for d in enumerate_disk(device):
271
if d['fstype'] == replace_fs and label_match is False:
272
# We found a device where we want to replace the FS
273
return ('/dev/%s' % d['name'], False)
275
if (d['fstype'] == fs_type and
276
((label_match and d['label'] == label) or not label_match)):
277
# If we find a matching device, we return that
278
return ('/dev/%s' % d['name'], True)
280
if d['type'] in valid_targets:
282
if d['type'] != 'disk' or d['fstype']:
283
raw_device_used = True
285
if d['type'] == 'disk':
286
# Skip the raw disk, its the default
289
elif not d['fstype']:
290
return ('/dev/%s' % d['name'], False)
292
if not raw_device_used:
293
return (device, False)
295
LOG.warn("Failed to find device during available device search.")
299
def is_disk_used(device):
301
Check if the device is currently used. Returns true if the device
302
has either a file system or a partition entry
303
is no filesystem found on the disk.
306
# If the child count is higher 1, then there are child nodes
307
# such as partition or device mapper nodes
308
if len(list(enumerate_disk(device))) > 1:
311
# If we see a file system, then its used
312
_, check_fstype, _ = check_fs(device)
319
def get_dyn_func(*args):
321
Call the appropriate function.
323
The first value is the template for function name
324
The second value is the template replacement
325
The remain values are passed to the function
327
For example: get_dyn_func("foo_%s", 'bar', 1, 2, 3,)
328
would call "foo_bar" with args of 1, 2, 3
331
raise Exception("Unable to determine dynamic funcation name")
333
func_name = (args[0] % args[1])
338
return globals()[func_name](*func_args)
340
return globals()[func_name]
343
raise Exception("No such function %s to call!" % func_name)
346
def get_mbr_hdd_size(device):
347
size_cmd = [SFDISK_CMD, '--show-size', device]
350
size, _err = util.subp(size_cmd)
351
except Exception as e:
352
raise Exception("Failed to get %s size\n%s" % (device, e))
354
return int(size.strip())
357
def get_gpt_hdd_size(device):
358
out, _ = util.subp([SGDISK_CMD, '-p', device])
359
return out.splitlines()[0].split()[2]
362
def get_hdd_size(table_type, device):
364
Returns the hard disk size.
365
This works with any disk type, including GPT.
367
return get_dyn_func("get_%s_hdd_size", table_type, device)
370
def check_partition_mbr_layout(device, layout):
372
Returns true if the partition layout matches the one on the disk
374
Layout should be a list of values. At this time, this only
375
verifies that the number of partitions and their labels is correct.
379
prt_cmd = [SFDISK_CMD, "-l", device]
381
out, _err = util.subp(prt_cmd, data="%s\n" % layout)
382
except Exception as e:
383
raise Exception("Error running partition command on %s\n%s" % (
387
for line in out.splitlines():
392
if device in _line[0]:
393
# We don't understand extended partitions yet
394
if _line[-1].lower() in ['extended', 'empty']:
397
# Find the partition types
399
for x in sorted(range(1, len(_line)), reverse=True):
400
if _line[x].isdigit() and _line[x] != '/':
401
type_label = _line[x]
404
found_layout.append(type_label)
408
def check_partition_gpt_layout(device, layout):
409
prt_cmd = [SGDISK_CMD, '-p', device]
411
out, _err = util.subp(prt_cmd)
412
except Exception as e:
413
raise Exception("Error running partition command on %s\n%s" % (
416
out_lines = iter(out.splitlines())
418
for line in out_lines:
419
if line.strip().startswith('Number'):
422
return [line.strip().split()[-1] for line in out_lines]
425
def check_partition_layout(table_type, device, layout):
427
See if the partition lay out matches.
429
This is future a future proofing function. In order
430
to add support for other disk layout schemes, add a
431
function called check_partition_%s_layout
433
found_layout = get_dyn_func(
434
"check_partition_%s_layout", table_type, device, layout)
436
if isinstance(layout, bool):
437
# if we are using auto partitioning, or "True" be happy
438
# if a single partition exists.
439
if layout and len(found_layout) >= 1:
444
if len(found_layout) != len(layout):
447
# This just makes sure that the number of requested
448
# partitions and the type labels are right
449
for x in range(1, len(layout) + 1):
450
if isinstance(layout[x - 1], tuple):
451
_, part_type = layout[x]
452
if int(found_layout[x]) != int(part_type):
459
def get_partition_mbr_layout(size, layout):
461
Calculate the layout of the partition table. Partition sizes
462
are defined as percentage values or a tuple of percentage and
468
Defines the first partition to be a size of 1/3 the disk,
469
while the remaining 2/3's will be of type Linux Swap.
472
if not isinstance(layout, list) and isinstance(layout, bool):
473
# Create a single partition
476
if ((len(layout) == 0 and isinstance(layout, list)) or
477
not isinstance(layout, list)):
478
raise Exception("Partition layout is invalid")
480
last_part_num = len(layout)
481
if last_part_num > 4:
482
raise Exception("Only simply partitioning is allowed.")
487
part_type = 83 # Default to Linux
491
if isinstance(part, list):
493
raise Exception("Partition was incorrectly defined: %s" % part)
494
percent, part_type = part
496
part_size = int((float(size) * (float(percent) / 100)) / 1024)
498
if part_num == last_part_num:
499
part_definition.append(",,%s" % part_type)
501
part_definition.append(",%s,%s" % (part_size, part_type))
503
sfdisk_definition = "\n".join(part_definition)
504
if len(part_definition) > 4:
505
raise Exception("Calculated partition definition is too big\n%s" %
508
return sfdisk_definition
511
def get_partition_gpt_layout(size, layout):
512
if isinstance(layout, bool):
513
return [(None, [0, 0])]
516
for partition in layout:
517
if isinstance(partition, list):
518
if len(partition) != 2:
520
"Partition was incorrectly defined: %s" % partition)
521
percent, partition_type = partition
524
partition_type = None
526
part_size = int(float(size) * (float(percent) / 100))
527
partition_specs.append((partition_type, [0, '+{}'.format(part_size)]))
529
# The last partition should use up all remaining space
530
partition_specs[-1][-1][-1] = 0
531
return partition_specs
534
def purge_disk_ptable(device):
535
# wipe the first and last megabyte of a disk (or file)
536
# gpt stores partition table both at front and at end.
538
start_len = 1024 * 1024
539
end_len = 1024 * 1024
540
with open(device, "rb+") as fp:
541
fp.write(null * (start_len))
542
fp.seek(-end_len, os.SEEK_END)
543
fp.write(null * end_len)
549
def purge_disk(device):
551
Remove parition table entries
554
# wipe any file systems first
555
for d in enumerate_disk(device):
556
if d['type'] not in ["disk", "crypt"]:
557
wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']]
559
LOG.info("Purging filesystem on /dev/%s" % d['name'])
560
util.subp(wipefs_cmd)
562
raise Exception("Failed FS purge of /dev/%s" % d['name'])
564
purge_disk_ptable(device)
567
def get_partition_layout(table_type, size, layout):
569
Call the appropriate function for creating the table
570
definition. Returns the table definition
572
This is a future proofing function. To add support for
573
other layouts, simply add a "get_partition_%s_layout"
576
return get_dyn_func("get_partition_%s_layout", table_type, size, layout)
579
def read_parttbl(device):
581
Use partprobe instead of 'udevadm'. Partprobe is the only
582
reliable way to probe the partition table.
584
blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device]
585
udev_cmd = [UDEVADM_CMD, 'settle']
588
util.subp(blkdev_cmd)
590
except Exception as e:
591
util.logexc(LOG, "Failed reading the partition table %s" % e)
594
def exec_mkpart_mbr(device, layout):
596
Break out of mbr partition to allow for future partition
599
# Create the partitions
600
prt_cmd = [SFDISK_CMD, "--Linux", "-uM", device]
602
util.subp(prt_cmd, data="%s\n" % layout)
603
except Exception as e:
604
raise Exception("Failed to partition device %s\n%s" % (device, e))
609
def exec_mkpart_gpt(device, layout):
611
util.subp([SGDISK_CMD, '-Z', device])
612
for index, (partition_type, (start, end)) in enumerate(layout):
614
util.subp([SGDISK_CMD,
615
'-n', '{}:{}:{}'.format(index, start, end), device])
616
if partition_type is not None:
619
'-t', '{}:{}'.format(index, partition_type), device])
621
LOG.warn("Failed to partition device %s" % device)
625
def exec_mkpart(table_type, device, layout):
627
Fetches the function for creating the table type.
628
This allows to dynamically find which function to call.
631
table_type: type of partition table to use
632
device: the device to work on
633
layout: layout definition specific to partition table
635
return get_dyn_func("exec_mkpart_%s", table_type, device, layout)
638
def mkpart(device, definition):
640
Creates the partition table.
643
definition: dictionary describing how to create the partition.
645
The following are supported values in the dict:
646
overwrite: Should the partition table be created regardless
647
of any pre-exisiting data?
648
layout: the layout of the partition table
649
table_type: Which partition table to use, defaults to MBR
650
device: the device to work on.
652
# ensure that we get a real device rather than a symbolic link
653
device = os.path.realpath(device)
655
LOG.debug("Checking values for %s definition" % device)
656
overwrite = definition.get('overwrite', False)
657
layout = definition.get('layout', False)
658
table_type = definition.get('table_type', 'mbr')
660
# Check if the default device is a partition or not
661
LOG.debug("Checking against default devices")
663
if (isinstance(layout, bool) and not layout) or not layout:
664
LOG.debug("Device is not to be partitioned, skipping")
665
return # Device is not to be partitioned
667
# This prevents you from overwriting the device
668
LOG.debug("Checking if device %s is a valid device", device)
669
if not is_device_valid(device):
670
raise Exception("Device %s is not a disk device!", device)
672
# Remove the partition table entries
673
if isinstance(layout, str) and layout.lower() == "remove":
674
LOG.debug("Instructed to remove partition table entries")
678
LOG.debug("Checking if device layout matches")
679
if check_partition_layout(table_type, device, layout):
680
LOG.debug("Device partitioning layout matches")
683
LOG.debug("Checking if device is safe to partition")
684
if not overwrite and (is_disk_used(device) or is_filesystem(device)):
685
LOG.debug("Skipping partitioning on configured device %s" % device)
688
LOG.debug("Checking for device size")
689
device_size = get_hdd_size(table_type, device)
691
LOG.debug("Calculating partition layout")
692
part_definition = get_partition_layout(table_type, device_size, layout)
693
LOG.debug(" Layout is: %s" % part_definition)
695
LOG.debug("Creating partition table on %s", device)
696
exec_mkpart(table_type, device, part_definition)
698
LOG.debug("Partition table created for %s", device)
701
def lookup_force_flag(fs):
703
A force flag might be -F or -F, this look it up
712
if 'ext' in fs.lower():
715
if fs.lower() in flags:
718
LOG.warn("Force flag for %s is unknown." % fs)
724
Create a file system on the device.
726
label: defines the label to use on the device
727
fs_cfg: defines how the filesystem is to look
728
The following values are required generally:
729
device: which device or cloud defined default_device
730
filesystem: which file system type
731
overwrite: indiscriminately create the file system
732
partition: when device does not define a partition,
733
setting this to a number will mean
734
device + partition. When set to 'auto', the
735
first free device or the first device which
736
matches both label and type will be used.
738
'any' means the first filesystem that matches
741
When 'cmd' is provided then no other parameter is required.
743
label = fs_cfg.get('label')
744
device = fs_cfg.get('device')
745
partition = str(fs_cfg.get('partition', 'any'))
746
fs_type = fs_cfg.get('filesystem')
747
fs_cmd = fs_cfg.get('cmd', [])
748
fs_opts = fs_cfg.get('extra_opts', [])
749
fs_replace = fs_cfg.get('replace_fs', False)
750
overwrite = fs_cfg.get('overwrite', False)
752
# ensure that we get a real device rather than a symbolic link
753
device = os.path.realpath(device)
755
# This allows you to define the default ephemeral or swap
756
LOG.debug("Checking %s against default devices", device)
758
if not partition or partition.isdigit():
759
# Handle manual definition of partition
760
if partition.isdigit():
761
device = "%s%s" % (device, partition)
762
LOG.debug("Manual request of partition %s for %s",
765
# Check to see if the fs already exists
766
LOG.debug("Checking device %s", device)
767
check_label, check_fstype, _ = check_fs(device)
768
LOG.debug("Device %s has %s %s", device, check_label, check_fstype)
770
if check_label == label and check_fstype == fs_type:
771
LOG.debug("Existing file system found at %s", device)
774
LOG.debug("Device %s has required file system", device)
777
LOG.warn("Destroying filesystem on %s", device)
780
LOG.debug("Device %s is cleared for formating", device)
782
elif partition and str(partition).lower() in ('auto', 'any'):
783
# For auto devices, we match if the filesystem does exist
785
LOG.debug("Identifying device to create %s filesytem on", label)
787
# any mean pick the first match on the device with matching fs_type
789
if partition.lower() == 'any':
792
device, reuse = find_device_node(device, fs_type=fs_type, label=label,
793
label_match=label_match,
794
replace_fs=fs_replace)
795
LOG.debug("Automatic device for %s identified as %s", odevice, device)
798
LOG.debug("Found filesystem match, skipping formating.")
801
if not reuse and fs_replace and device:
802
LOG.debug("Replacing file system on %s as instructed." % device)
805
LOG.debug("No device aviable that matches request. "
806
"Skipping fs creation for %s", fs_cfg)
808
elif not partition or str(partition).lower() == 'none':
809
LOG.debug("Using the raw device to place filesystem %s on" % label)
812
LOG.debug("Error in device identification handling.")
815
LOG.debug("File system %s will be created on %s", label, device)
817
# Make sure the device is defined
819
LOG.warn("Device is not known: %s", device)
822
# Check that we can create the FS
823
if not (fs_type or fs_cmd):
824
raise Exception("No way to create filesystem '%s'. fs_type or fs_cmd "
825
"must be set.", label)
827
# Create the commands
829
fs_cmd = fs_cfg['cmd'] % {
831
'filesystem': fs_type,
835
# Find the mkfs command
836
mkfs_cmd = util.which("mkfs.%s" % fs_type)
838
mkfs_cmd = util.which("mk%s" % fs_type)
841
LOG.warn("Cannot create fstype '%s'. No mkfs.%s command", fs_type,
845
fs_cmd = [mkfs_cmd, device]
848
fs_cmd.extend(["-L", label])
850
# File systems that support the -F flag
851
if overwrite or device_type(device) == "disk":
852
fs_cmd.append(lookup_force_flag(fs_type))
854
# Add the extends FS options
856
fs_cmd.extend(fs_opts)
858
LOG.debug("Creating file system %s on %s", label, device)
859
LOG.debug(" Using cmd: %s", " ".join(fs_cmd))
862
except Exception as e:
863
raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e))
added
removed
cloudinit/config/cc_disk_setup.py
