1
# Copyright (C) 2016 Canonical Ltd.
3
# Author: Christian Ehrhardt <christian.ehrhardt@canonical.com>
5
# Curtin is free software: you can redistribute it and/or modify it under
6
# the terms of the GNU Affero General Public License as published by the
7
# Free Software Foundation, either version 3 of the License, or (at your
8
# option) any later version.
10
# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
12
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
15
# You should have received a copy of the GNU Affero General Public License
16
# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
19
Handle the setup of apt related tasks like proxies, mirrors, repositories.
29
from curtin.log import LOG
30
from curtin import (config, util, gpg)
32
from . import populate_one_subcmd
34
# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
35
ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
37
# place where apt stores cached repository data
38
APT_LISTS = "/var/lib/apt/lists"
40
# Files to store proxy information
41
APT_CONFIG_FN = "/etc/apt/apt.conf.d/94curtin-config"
42
APT_PROXY_FN = "/etc/apt/apt.conf.d/90curtin-aptproxy"
44
# Default keyserver to use
45
DEFAULT_KEYSERVER = "keyserver.ubuntu.com"
47
# Default archive mirrors
48
PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/",
49
"SECURITY": "http://security.ubuntu.com/ubuntu/"}
50
PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports",
51
"SECURITY": "http://ports.ubuntu.com/ubuntu-ports"}
52
PRIMARY_ARCHES = ['amd64', 'i386']
53
PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']
56
def get_default_mirrors(arch=None):
57
"""returns the default mirrors for the target. These depend on the
58
architecture, for more see:
59
https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
61
arch = util.get_architecture()
62
if arch in PRIMARY_ARCHES:
63
return PRIMARY_ARCH_MIRRORS.copy()
64
if arch in PORTS_ARCHES:
65
return PORTS_MIRRORS.copy()
66
raise ValueError("No default mirror known for arch %s" % arch)
69
def handle_apt(cfg, target=None):
71
process the config for apt_config. This can be called from
72
curthooks if a global apt config was provided or via the "apt"
75
release = util.lsb_release(target=target)['codename']
76
arch = util.get_architecture(target)
77
mirrors = find_apt_mirror_info(cfg, arch)
78
LOG.debug("Apt Mirror info: %s", mirrors)
80
apply_debconf_selections(cfg, target)
82
if not config.value_as_boolean(cfg.get('preserve_sources_list',
84
generate_sources_list(cfg, release, mirrors, target)
85
rename_apt_lists(mirrors, target)
88
apply_apt_proxy_config(cfg, target + APT_PROXY_FN,
89
target + APT_CONFIG_FN)
90
except (IOError, OSError):
91
LOG.exception("Failed to apply proxy or apt config info:")
93
# Process 'apt_source -> sources {dict}'
96
params['RELEASE'] = release
97
params['MIRROR'] = mirrors["MIRROR"]
100
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
102
matcher = re.compile(matchcfg).search
104
add_apt_sources(cfg['sources'], target,
105
template_params=params, aa_repo_match=matcher)
108
def debconf_set_selections(selections, target=None):
109
util.subp(['debconf-set-selections'], data=selections, target=target,
113
def dpkg_reconfigure(packages, target=None):
114
# For any packages that are already installed, but have preseed data
115
# we populate the debconf database, but the filesystem configuration
116
# would be preferred on a subsequent dpkg-reconfigure.
117
# so, what we have to do is "know" information about certain packages
118
# to unconfigure them.
122
if pkg in CONFIG_CLEANERS:
123
LOG.debug("unconfiguring %s", pkg)
124
CONFIG_CLEANERS[pkg](target)
125
to_config.append(pkg)
127
unhandled.append(pkg)
130
LOG.warn("The following packages were installed and preseeded, "
131
"but cannot be unconfigured: %s", unhandled)
134
util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
135
list(to_config), data=None, target=target, capture=True)
138
def apply_debconf_selections(cfg, target=None):
139
"""apply_debconf_selections - push content to debconf"""
140
# debconf_selections:
142
# cloud-init cloud-init/datasources multiselect MAAS
143
# set2: pkg pkg/value string bar
144
selsets = cfg.get('debconf_selections')
146
LOG.debug("debconf_selections was not set in config")
149
selections = '\n'.join(
150
[selsets[key] for key in sorted(selsets.keys())])
151
debconf_set_selections(selections.encode() + b"\n", target=target)
153
# get a complete list of packages listed in input
155
for key, content in selsets.items():
156
for line in content.splitlines():
157
if line.startswith("#"):
159
pkg = re.sub(r"[:\s].*", "", line)
162
pkgs_installed = util.get_installed_packages(target)
164
LOG.debug("pkgs_cfgd: %s", pkgs_cfgd)
165
LOG.debug("pkgs_installed: %s", pkgs_installed)
166
need_reconfig = pkgs_cfgd.intersection(pkgs_installed)
168
if len(need_reconfig) == 0:
169
LOG.debug("no need for reconfig")
172
dpkg_reconfigure(need_reconfig, target=target)
175
def clean_cloud_init(target):
176
"""clean out any local cloud-init config"""
178
util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
180
LOG.debug("cleaning cloud-init config from: %s", flist)
181
for dpkg_cfg in flist:
185
def mirrorurl_to_apt_fileprefix(mirror):
186
""" mirrorurl_to_apt_fileprefix
187
Convert a mirror url to the file prefix used by apt on disk to
188
store cache information for that mirror.
192
- convert in string / to _
195
if string.endswith("/"):
196
string = string[0:-1]
197
pos = string.find("://")
199
string = string[pos + 3:]
200
string = string.replace("/", "_")
204
def rename_apt_lists(new_mirrors, target=None):
205
"""rename_apt_lists - rename apt lists to preserve old cache data"""
206
default_mirrors = get_default_mirrors(util.get_architecture(target))
208
pre = util.target_path(target, APT_LISTS)
209
for (name, omirror) in default_mirrors.items():
210
nmirror = new_mirrors.get(name)
214
oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
215
nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
216
if oprefix == nprefix:
219
for filename in glob.glob("%s_*" % oprefix):
220
newname = "%s%s" % (nprefix, filename[olen:])
221
LOG.debug("Renaming apt list %s to %s", filename, newname)
223
os.rename(filename, newname)
225
# since this is a best effort task, warn with but don't fail
226
LOG.warn("Failed to rename apt list:", exc_info=True)
229
def mirror_to_placeholder(tmpl, mirror, placeholder):
230
""" mirror_to_placeholder
231
replace the specified mirror in a template with a placeholder string
232
Checks for existance of the expected mirror and warns if not found
234
if mirror not in tmpl:
235
LOG.warn("Expected mirror '%s' not found in: %s", mirror, tmpl)
236
return tmpl.replace(mirror, placeholder)
239
def map_known_suites(suite):
240
"""there are a few default names which will be auto-extended.
241
This comes at the inability to use those names literally as suites,
242
but on the other hand increases readability of the cfg quite a lot"""
243
mapping = {'updates': '$RELEASE-updates',
244
'backports': '$RELEASE-backports',
245
'security': '$RELEASE-security',
246
'proposed': '$RELEASE-proposed',
247
'release': '$RELEASE'}
249
retsuite = mapping[suite]
255
def disable_suites(disabled, src, release):
256
"""reads the config for suites to be disabled and removes those
262
for suite in disabled:
263
suite = map_known_suites(suite)
264
releasesuite = util.render_string(suite, {'RELEASE': release})
265
LOG.debug("Disabling suite %s as %s", suite, releasesuite)
268
for line in retsrc.splitlines(True):
269
if line.startswith("#"):
273
# sources.list allow options in cols[1] which can have spaces
274
# so the actual suite can be [2] or later. example:
275
# deb [ arch=amd64,armel k=v ] http://example.com/debian
279
if cols[1].startswith("["):
282
if col.endswith("]"):
285
if cols[pcol] == releasesuite:
286
line = '# suite disabled by curtin: %s' % line
293
def generate_sources_list(cfg, release, mirrors, target=None):
294
""" generate_sources_list
295
create a source.list file based on a custom or default template
296
by replacing mirrors and release in the template
298
default_mirrors = get_default_mirrors(util.get_architecture(target))
299
aptsrc = "/etc/apt/sources.list"
300
params = {'RELEASE': release}
302
params[k] = mirrors[k]
304
tmpl = cfg.get('sources_list', None)
306
LOG.info("No custom template provided, fall back to modify"
307
"mirrors in %s on the target system", aptsrc)
308
tmpl = util.load_file(util.target_path(target, aptsrc))
309
# Strategy if no custom template was provided:
310
# - Only replacing mirrors
311
# - no reason to replace "release" as it is from target anyway
312
# - The less we depend upon, the more stable this is against changes
313
# - warn if expected original content wasn't found
314
tmpl = mirror_to_placeholder(tmpl, default_mirrors['PRIMARY'],
316
tmpl = mirror_to_placeholder(tmpl, default_mirrors['SECURITY'],
319
orig = util.target_path(target, aptsrc)
320
if os.path.exists(orig):
321
os.rename(orig, orig + ".curtin.old")
323
rendered = util.render_string(tmpl, params)
324
disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
325
util.write_file(util.target_path(target, aptsrc), disabled, mode=0o644)
327
# protect the just generated sources.list from cloud-init
328
cloudfile = "/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg"
329
# this has to work with older cloud-init as well, so use old key
330
cloudconf = yaml.dump({'apt_preserve_sources_list': True}, indent=1)
332
util.write_file(util.target_path(target, cloudfile),
333
cloudconf, mode=0o644)
335
LOG.exception("Failed to protect source.list from cloud-init in (%s)",
336
util.target_path(target, cloudfile))
340
def add_apt_key_raw(key, target=None):
342
actual adding of a key as defined in key argument
345
LOG.debug("Adding key:\n'%s'", key)
347
util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
348
except util.ProcessExecutionError:
349
LOG.exception("failed to add apt GPG Key to apt keyring")
353
def add_apt_key(ent, target=None):
355
Add key to the system as defined in ent (if any).
356
Supports raw keys or keyid's
357
The latter will as a first step fetched to get the raw key
359
if 'keyid' in ent and 'key' not in ent:
360
keyserver = DEFAULT_KEYSERVER
361
if 'keyserver' in ent:
362
keyserver = ent['keyserver']
364
ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver,
365
retries=(1, 2, 5, 10))
368
add_apt_key_raw(ent['key'], target)
371
def add_apt_sources(srcdict, target=None, template_params=None,
374
add entries in /etc/apt/sources.list.d for each abbreviated
375
sources.list entry in 'srcdict'. When rendering template, also
376
include the values in dictionary searchList
378
if template_params is None:
381
if aa_repo_match is None:
382
raise ValueError('did not get a valid repo matcher')
384
if not isinstance(srcdict, dict):
385
raise TypeError('unknown apt format: %s' % (srcdict))
387
for filename in srcdict:
388
ent = srcdict[filename]
389
if 'filename' not in ent:
390
ent['filename'] = filename
392
add_apt_key(ent, target)
394
if 'source' not in ent:
396
source = ent['source']
397
source = util.render_string(source, template_params)
399
if not ent['filename'].startswith("/"):
400
ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
402
if not ent['filename'].endswith(".list"):
403
ent['filename'] += ".list"
405
if aa_repo_match(source):
406
with util.ChrootableTarget(
407
target, sys_resolvconf=True) as in_chroot:
409
in_chroot.subp(["add-apt-repository", source],
410
retries=(1, 2, 5, 10))
411
except util.ProcessExecutionError:
412
LOG.exception("add-apt-repository failed.")
416
sourcefn = util.target_path(target, ent['filename'])
418
contents = "%s\n" % (source)
419
util.write_file(sourcefn, contents, omode="a")
420
except IOError as detail:
421
LOG.exception("failed write to file %s: %s", sourcefn, detail)
424
util.apt_update(target=target, force=True,
425
comment="apt-source changed config")
430
def search_for_mirror(candidates):
432
Search through a list of mirror urls for one that works
433
This needs to return quickly.
435
if candidates is None:
438
LOG.debug("search for mirror in candidates: '%s'", candidates)
439
for cand in candidates:
441
if util.is_resolvable_url(cand):
442
LOG.debug("found working mirror: '%s'", cand)
449
def update_mirror_info(pmirror, smirror, arch):
450
"""sets security mirror to primary if not defined.
451
returns defaults if no mirrors are defined"""
452
if pmirror is not None:
455
return {'PRIMARY': pmirror,
457
return get_default_mirrors(arch)
460
def get_arch_mirrorconfig(cfg, mirrortype, arch):
461
"""out of a list of potential mirror configurations select
462
and return the one matching the architecture (or default)"""
463
# select the mirror specification (if-any)
464
mirror_cfg_list = cfg.get(mirrortype, None)
465
if mirror_cfg_list is None:
468
# select the specification matching the target arch
470
for mirror_cfg_elem in mirror_cfg_list:
471
arches = mirror_cfg_elem.get("arches")
473
return mirror_cfg_elem
474
if "default" in arches:
475
default = mirror_cfg_elem
479
def get_mirror(cfg, mirrortype, arch):
480
"""pass the three potential stages of mirror specification
481
returns None is neither of them found anything otherwise the first
483
mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch)
488
mirror = mcfg.get("uri", None)
490
# fallback to search if specified
492
# list of mirrors to try to resolve
493
mirror = search_for_mirror(mcfg.get("search", None))
498
def find_apt_mirror_info(cfg, arch=None):
499
"""find_apt_mirror_info
500
find an apt_mirror given the cfg provided.
501
It can check for separate config of primary and security mirrors
502
If only primary is given security is assumed to be equal to primary
503
If the generic apt_mirror is given that is defining for both
507
arch = util.get_architecture()
508
LOG.debug("got arch for mirror selection: %s", arch)
509
pmirror = get_mirror(cfg, "primary", arch)
510
LOG.debug("got primary mirror: %s", pmirror)
511
smirror = get_mirror(cfg, "security", arch)
512
LOG.debug("got security mirror: %s", smirror)
514
# Note: curtin has no cloud-datasource fallback
516
mirror_info = update_mirror_info(pmirror, smirror, arch)
518
# less complex replacements use only MIRROR, derive from primary
519
mirror_info["MIRROR"] = mirror_info["PRIMARY"]
524
def apply_apt_proxy_config(cfg, proxy_fname, config_fname):
525
"""apply_apt_proxy_config
526
Applies any apt*proxy config from if specified
528
# Set up any apt proxy
529
cfgs = (('proxy', 'Acquire::http::Proxy "%s";'),
530
('http_proxy', 'Acquire::http::Proxy "%s";'),
531
('ftp_proxy', 'Acquire::ftp::Proxy "%s";'),
532
('https_proxy', 'Acquire::https::Proxy "%s";'))
534
proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)]
536
LOG.debug("write apt proxy info to %s", proxy_fname)
537
util.write_file(proxy_fname, '\n'.join(proxies) + '\n')
538
elif os.path.isfile(proxy_fname):
539
util.del_file(proxy_fname)
540
LOG.debug("no apt proxy configured, removed %s", proxy_fname)
542
if cfg.get('conf', None):
543
LOG.debug("write apt config info to %s", config_fname)
544
util.write_file(config_fname, cfg.get('conf'))
545
elif os.path.isfile(config_fname):
546
util.del_file(config_fname)
547
LOG.debug("no apt config configured, removed %s", config_fname)
550
def apt_command(args):
551
""" Main entry point for curtin apt-config standalone command
552
This does not read the global config as handled by curthooks, but
553
instead one can specify a different "target" and a new cfg via --config
555
cfg = config.load_command_config(args, {})
557
if args.target is not None:
560
state = util.load_command_environment()
561
target = state['target']
564
sys.stderr.write("Unable to find target. "
565
"Use --target or set TARGET_MOUNT_POINT\n")
568
apt_cfg = cfg.get("apt")
569
# if no apt config section is available, do nothing
570
if apt_cfg is not None:
571
LOG.debug("Handling apt to target %s with config %s",
574
with util.ChrootableTarget(target, sys_resolvconf=True):
575
handle_apt(apt_cfg, target)
576
except (RuntimeError, TypeError, ValueError, IOError):
577
LOG.exception("Failed to configure apt features '%s'", apt_cfg)
580
LOG.info("No apt config provided, skipping")
585
def translate_old_apt_features(cfg):
586
"""translate the few old apt related features into the new config format"""
587
predef_apt_cfg = cfg.get("apt")
588
if predef_apt_cfg is None:
590
predef_apt_cfg = cfg.get("apt")
592
if cfg.get('apt_proxy') is not None:
593
if predef_apt_cfg.get('proxy') is not None:
594
msg = ("Error in apt_proxy configuration: "
595
"old and new format of apt features "
596
"are mutually exclusive")
598
raise ValueError(msg)
600
cfg['apt']['proxy'] = cfg.get('apt_proxy')
601
LOG.debug("Transferred %s into new format: %s", cfg.get('apt_proxy'),
605
if cfg.get('apt_mirrors') is not None:
606
if predef_apt_cfg.get('mirrors') is not None:
607
msg = ("Error in apt_mirror configuration: "
608
"old and new format of apt features "
609
"are mutually exclusive")
611
raise ValueError(msg)
613
old = cfg.get('apt_mirrors')
614
cfg['apt']['primary'] = [{"arches": ["default"],
615
"uri": old.get('ubuntu_archive')}]
616
cfg['apt']['security'] = [{"arches": ["default"],
617
"uri": old.get('ubuntu_security')}]
618
LOG.debug("Transferred %s into new format: %s", cfg.get('apt_mirror'),
620
del cfg['apt_mirrors']
621
# to work this also needs to disable the default protection
622
psl = predef_apt_cfg.get('preserve_sources_list')
624
if config.value_as_boolean(psl) is True:
625
msg = ("Error in apt_mirror configuration: "
626
"apt_mirrors and preserve_sources_list: True "
627
"are mutually exclusive")
629
raise ValueError(msg)
630
cfg['apt']['preserve_sources_list'] = False
632
if cfg.get('debconf_selections') is not None:
633
if predef_apt_cfg.get('debconf_selections') is not None:
634
msg = ("Error in debconf_selections configuration: "
635
"old and new format of apt features "
636
"are mutually exclusive")
638
raise ValueError(msg)
640
selsets = cfg.get('debconf_selections')
641
cfg['apt']['debconf_selections'] = selsets
642
LOG.info("Transferred %s into new format: %s",
643
cfg.get('debconf_selections'),
645
del cfg['debconf_selections']
651
((('-c', '--config'),
652
{'help': 'read configuration from cfg', 'action': util.MergedCmdAppend,
653
'metavar': 'FILE', 'type': argparse.FileType("rb"),
654
'dest': 'cfgopts', 'default': []}),
656
{'help': 'chroot to target. default is env[TARGET_MOUNT_POINT]',
657
'action': 'store', 'metavar': 'TARGET',
658
'default': os.environ.get('TARGET_MOUNT_POINT')}),)
662
def POPULATE_SUBCMD(parser):
663
"""Populate subcommand option parsing for apt-config"""
664
populate_one_subcmd(parser, CMD_ARGUMENTS, apt_command)
668
'cloud-init': clean_cloud_init,
671
# vi: ts=4 expandtab syntax=python
added
removed
curtin/commands/apt_config.py
