« back to all changes in this revision
Viewing changes to curtin/util.py
- browse files at revision 1.1.40
- compare with another revision
- download diff
- download tarball
- view history from revision 1.1.40
- Committer: Scott Moser
- Date: 2016-08-05 20:47:14 UTC
- mto: (58.1.1 pkg)
- mto: This revision was merged to the branch mainline in revision 56.
- Revision ID: smoser@ubuntu.com-20160805204714-f6j1k61cli5uf614
Tags: upstream-0.1.0~bzr415
Import upstream version 0.1.0~bzr415
- files added:
- curtin/commands/apt_config.py
- files modified:
- curtin/__init__.py
added
removed
curtin/util.py
16
16
# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
17
17
18
18
import argparse
19
import collections
19
20
import errno
20
21
import glob
21
22
import json
22
23
import os
23
24
import platform
25
import re
24
26
import shutil
27
import socket
25
28
import subprocess
26
29
import stat
27
30
import sys
28
31
import tempfile
29
32
import time
30
33
34
# avoid the dependency to python3-six as used in cloud-init
35
try:
36
from urlparse import urlparse
37
except ImportError:
38
# python3
39
# avoid triggering pylint, https://github.com/PyCQA/pylint/issues/769
40
# pylint:disable=import-error,no-name-in-module
41
from urllib.parse import urlparse
42
43
try:
44
string_types = (basestring,)
45
except NameError:
46
string_types = (str,)
47
31
48
from .log import LOG
32
49
33
50
_INSTALLED_HELPERS_PATH = '/usr/lib/curtin/helpers'
35
52
36
53
_LSB_RELEASE = {}
37
54
55
_DNS_REDIRECT_IP = None
56
57
# matcher used in template rendering functions
58
BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)')
59
38
60
39
61
def _subp(args, data=None, rcs=None, env=None, capture=False, shell=False,
40
logstring=False, decode="replace"):
62
logstring=False, decode="replace", target=None):
41
63
if rcs is None:
42
64
rcs = [0]
43
65
44
66
devnull_fp = None
45
67
try:
68
if target_path(target) != "/":
69
args = ['chroot', target] + list(args)
70
46
71
if not logstring:
47
72
LOG.debug(("Running command %s with allowed return codes %s"
48
73
" (shell=%s, capture=%s)"), args, rcs, shell, capture)
311
336
'done',
312
337
''])
313
338
314
fpath = os.path.join(target, "usr/sbin/policy-rc.d")
339
fpath = target_path(target, "/usr/sbin/policy-rc.d")
315
340
316
341
if os.path.isfile(fpath):
317
342
return False
322
347
323
348
def undisable_daemons_in_root(target):
324
349
try:
325
os.unlink(os.path.join(target, "usr/sbin/policy-rc.d"))
350
os.unlink(target_path(target, "/usr/sbin/policy-rc.d"))
326
351
except OSError as e:
327
352
if e.errno != errno.ENOENT:
328
353
raise
334
359
def __init__(self, target, allow_daemons=False, sys_resolvconf=True):
335
360
if target is None:
336
361
target = "/"
337
self.target = os.path.abspath(target)
362
self.target = target_path(target)
338
363
self.mounts = ["/dev", "/proc", "/sys"]
339
364
self.umounts = []
340
365
self.disabled_daemons = False
344
369
345
370
def __enter__(self):
346
371
for p in self.mounts:
347
tpath = os.path.join(self.target, p[1:])
372
tpath = target_path(self.target, p)
348
373
if do_mount(p, tpath, opts='--bind'):
349
374
self.umounts.append(tpath)
350
375
351
376
if not self.allow_daemons:
352
377
self.disabled_daemons = disable_daemons_in_root(self.target)
353
378
354
target_etc = os.path.join(self.target, "etc")
379
rconf = target_path(self.target, "/etc/resolv.conf")
380
target_etc = os.path.dirname(rconf)
355
381
if self.target != "/" and os.path.isdir(target_etc):
356
382
# never muck with resolv.conf on /
357
383
rconf = os.path.join(target_etc, "resolv.conf")
358
384
rtd = None
359
385
try:
360
rtd = tempfile.mkdtemp(dir=os.path.dirname(rconf))
386
rtd = tempfile.mkdtemp(dir=target_etc)
361
387
tmp = os.path.join(rtd, "resolv.conf")
362
388
os.rename(rconf, tmp)
363
389
self.rconf_d = rtd
375
401
undisable_daemons_in_root(self.target)
376
402
377
403
# if /dev is to be unmounted, udevadm settle (LP: #1462139)
378
if os.path.join(self.target, "dev") in self.umounts:
404
if target_path(self.target, "/dev") in self.umounts:
379
405
subp(['udevadm', 'settle'])
380
406
381
407
for p in reversed(self.umounts):
382
408
do_umount(p)
383
409
384
rconf = os.path.join(self.target, "etc", "resolv.conf")
410
rconf = target_path(self.target, "/etc/resolv.conf")
385
411
if self.sys_resolvconf and self.rconf_d:
386
412
os.rename(os.path.join(self.rconf_d, "resolv.conf"), rconf)
387
413
shutil.rmtree(self.rconf_d)
388
414
415
def subp(self, *args, **kwargs):
416
kwargs['target'] = self.target
417
return subp(*args, **kwargs)
389
418
390
class RunInChroot(ChrootableTarget):
391
def __call__(self, args, **kwargs):
392
if self.target != "/":
393
chroot = ["chroot", self.target]
394
else:
395
chroot = []
396
return subp(chroot + args, **kwargs)
419
def path(self, path):
420
return target_path(self.target, path)
397
421
398
422
399
423
def is_exe(fpath):
402
426
403
427
404
428
def which(program, search=None, target=None):
405
if target is None or os.path.realpath(target) == "/":
406
target = "/"
429
target = target_path(target)
407
430
408
431
if os.path.sep in program:
409
432
# if program had a '/' in it, then do not search PATH
410
433
# 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls
411
434
# so effectively we set cwd to / (or target)
412
if is_exe(os.path.sep.join((target, program,))):
435
if is_exe(target_path(target, program)):
413
436
return program
414
437
415
438
if search is None:
424
447
search = [os.path.abspath(p) for p in search]
425
448
426
449
for path in search:
427
if is_exe(os.path.sep.join((target, path, program,))):
428
return os.path.sep.join((path, program,))
450
ppath = os.path.sep.join((path, program))
451
if is_exe(target_path(target, ppath)):
452
return ppath
429
453
430
454
return None
431
455
467
491
468
492
469
493
def get_architecture(target=None):
470
chroot = []
471
if target is not None:
472
chroot = ['chroot', target]
473
out, _ = subp(chroot + ['dpkg', '--print-architecture'],
474
capture=True)
494
out, _ = subp(['dpkg', '--print-architecture'], capture=True,
495
target=target)
475
496
return out.strip()
476
497
477
498
478
499
def has_pkg_available(pkg, target=None):
479
chroot = []
480
if target is not None:
481
chroot = ['chroot', target]
482
out, _ = subp(chroot + ['apt-cache', 'pkgnames'], capture=True)
500
out, _ = subp(['apt-cache', 'pkgnames'], capture=True, target=target)
483
501
for item in out.splitlines():
484
502
if pkg == item.strip():
485
503
return True
486
504
return False
487
505
488
506
507
def get_installed_packages(target=None):
508
(out, _) = subp(['dpkg-query', '--list'], target=target, capture=True)
509
510
pkgs_inst = set()
511
for line in out.splitlines():
512
try:
513
(state, pkg, other) = line.split(None, 2)
514
except ValueError:
515
continue
516
if state.startswith("hi") or state.startswith("ii"):
517
pkgs_inst.add(re.sub(":.*", "", pkg))
518
519
return pkgs_inst
520
521
489
522
def has_pkg_installed(pkg, target=None):
490
chroot = []
491
if target is not None:
492
chroot = ['chroot', target]
493
523
try:
494
out, _ = subp(chroot + ['dpkg-query', '--show', '--showformat',
495
'${db:Status-Abbrev}', pkg],
496
capture=True)
524
out, _ = subp(['dpkg-query', '--show', '--showformat',
525
'${db:Status-Abbrev}', pkg],
526
capture=True, target=target)
497
527
return out.rstrip() == "ii"
498
528
except ProcessExecutionError:
499
529
return False
542
572
"""Use dpkg-query to extract package pkg's version string
543
573
and parse the version string into a dictionary
544
574
"""
545
chroot = []
546
if target is not None:
547
chroot = ['chroot', target]
548
575
try:
549
out, _ = subp(chroot + ['dpkg-query', '--show', '--showformat',
550
'${Version}', pkg],
551
capture=True)
576
out, _ = subp(['dpkg-query', '--show', '--showformat',
577
'${Version}', pkg], capture=True, target=target)
552
578
raw = out.rstrip()
553
579
return parse_dpkg_version(raw, name=pkg, semx=semx)
554
580
except ProcessExecutionError:
600
626
if comment.endswith("\n"):
601
627
comment = comment[:-1]
602
628
603
marker = os.path.join(target, marker)
629
marker = target_path(target, marker)
604
630
# if marker exists, check if there are files that would make it obsolete
605
listfiles = [os.path.join(target, "etc/apt/sources.list")]
631
listfiles = [target_path(target, "/etc/apt/sources.list")]
606
632
listfiles += glob.glob(
607
os.path.join(target, "etc/apt/sources.list.d/*.list"))
633
target_path(target, "etc/apt/sources.list.d/*.list"))
608
634
609
635
if os.path.exists(marker) and not force:
610
636
if len(find_newer(marker, listfiles)) == 0:
612
638
613
639
restore_perms = []
614
640
615
abs_tmpdir = tempfile.mkdtemp(dir=os.path.join(target, 'tmp'))
641
abs_tmpdir = tempfile.mkdtemp(dir=target_path(target, "/tmp"))
616
642
try:
617
643
abs_slist = abs_tmpdir + "/sources.list"
618
644
abs_slistd = abs_tmpdir + "/sources.list.d"
621
647
ch_slistd = ch_tmpdir + "/sources.list.d"
622
648
623
649
# this file gets executed on apt-get update sometimes. (LP: #1527710)
624
motd_update = os.path.join(
625
target, "usr/lib/update-notifier/update-motd-updates-available")
650
motd_update = target_path(
651
target, "/usr/lib/update-notifier/update-motd-updates-available")
626
652
pmode = set_unexecutable(motd_update)
627
653
if pmode is not None:
628
654
restore_perms.append((motd_update, pmode),)
647
673
'update']
648
674
649
675
# do not using 'run_apt_command' so we can use 'retries' to subp
650
with RunInChroot(target, allow_daemons=True) as inchroot:
651
inchroot(update_cmd, env=env, retries=retries)
676
with ChrootableTarget(target, allow_daemons=True) as inchroot:
677
inchroot.subp(update_cmd, env=env, retries=retries)
652
678
finally:
653
679
for fname, perms in restore_perms:
654
680
os.chmod(fname, perms)
685
711
return env, cmd
686
712
687
713
apt_update(target, env=env, comment=' '.join(cmd))
688
ric = RunInChroot(target, allow_daemons=allow_daemons)
689
with ric as inchroot:
690
return inchroot(cmd, env=env)
714
with ChrootableTarget(target, allow_daemons=allow_daemons) as inchroot:
715
return inchroot.subp(cmd, env=env)
691
716
692
717
693
718
def system_upgrade(aptopts=None, target=None, env=None, allow_daemons=False):
716
741
"""
717
742
Look for "hook" in "target" and run it
718
743
"""
719
target_hook = os.path.join(target, 'curtin', hook)
744
target_hook = target_path(target, '/curtin/' + hook)
720
745
if os.path.isfile(target_hook):
721
746
LOG.debug("running %s" % target_hook)
722
747
subp([target_hook])
846
871
return (isinstance(exc, IOError) and exc.errno == errno.ENOENT)
847
872
848
873
849
def lsb_release():
874
def _lsb_release(target=None):
850
875
fmap = {'Codename': 'codename', 'Description': 'description',
851
876
'Distributor ID': 'id', 'Release': 'release'}
877
878
data = {}
879
try:
880
out, _ = subp(['lsb_release', '--all'], capture=True, target=target)
881
for line in out.splitlines():
882
fname, _, val = line.partition(":")
883
if fname in fmap:
884
data[fmap[fname]] = val.strip()
885
missing = [k for k in fmap.values() if k not in data]
886
if len(missing):
887
LOG.warn("Missing fields in lsb_release --all output: %s",
888
','.join(missing))
889
890
except ProcessExecutionError as err:
891
LOG.warn("Unable to get lsb_release --all: %s", err)
892
data = {v: "UNAVAILABLE" for v in fmap.values()}
893
894
return data
895
896
897
def lsb_release(target=None):
898
if target_path(target) != "/":
899
# do not use or update cache if target is provided
900
return _lsb_release(target)
901
852
902
global _LSB_RELEASE
853
903
if not _LSB_RELEASE:
854
data = {}
855
try:
856
out, err = subp(['lsb_release', '--all'], capture=True)
857
for line in out.splitlines():
858
fname, tok, val = line.partition(":")
859
if fname in fmap:
860
data[fmap[fname]] = val.strip()
861
missing = [k for k in fmap.values() if k not in data]
862
if len(missing):
863
LOG.warn("Missing fields in lsb_release --all output: %s",
864
','.join(missing))
865
866
except ProcessExecutionError as e:
867
LOG.warn("Unable to get lsb_release --all: %s", e)
868
data = {v: "UNAVAILABLE" for v in fmap.values()}
869
904
data = _lsb_release()
870
905
_LSB_RELEASE.update(data)
871
906
return _LSB_RELEASE
872
907
895
930
}
896
931
return platform2arch.get(platform.machine(), platform.machine())
897
932
933
934
def basic_template_render(content, params):
935
"""This does simple replacement of bash variable like templates.
936
937
It identifies patterns like ${a} or $a and can also identify patterns like
938
${a.b} or $a.b which will look for a key 'b' in the dictionary rooted
939
by key 'a'.
940
"""
941
942
def replacer(match):
943
""" replacer
944
replacer used in regex match to replace content
945
"""
946
# Only 1 of the 2 groups will actually have a valid entry.
947
name = match.group(1)
948
if name is None:
949
name = match.group(2)
950
if name is None:
951
raise RuntimeError("Match encountered but no valid group present")
952
path = collections.deque(name.split("."))
953
selected_params = params
954
while len(path) > 1:
955
key = path.popleft()
956
if not isinstance(selected_params, dict):
957
raise TypeError("Can not traverse into"
958
" non-dictionary '%s' of type %s while"
959
" looking for subkey '%s'"
960
% (selected_params,
961
selected_params.__class__.__name__,
962
key))
963
selected_params = selected_params[key]
964
key = path.popleft()
965
if not isinstance(selected_params, dict):
966
raise TypeError("Can not extract key '%s' from non-dictionary"
967
" '%s' of type %s"
968
% (key, selected_params,
969
selected_params.__class__.__name__))
970
return str(selected_params[key])
971
972
return BASIC_MATCHER.sub(replacer, content)
973
974
975
def render_string(content, params):
976
""" render_string
977
render a string following replacement rules as defined in
978
basic_template_render returning the string
979
"""
980
if not params:
981
params = {}
982
return basic_template_render(content, params)
983
984
985
def is_resolvable(name):
986
"""determine if a url is resolvable, return a boolean
987
This also attempts to be resilent against dns redirection.
988
989
Note, that normal nsswitch resolution is used here. So in order
990
to avoid any utilization of 'search' entries in /etc/resolv.conf
991
we have to append '.'.
992
993
The top level 'invalid' domain is invalid per RFC. And example.com
994
should also not exist. The random entry will be resolved inside
995
the search list.
996
"""
997
global _DNS_REDIRECT_IP
998
if _DNS_REDIRECT_IP is None:
999
badips = set()
1000
badnames = ("does-not-exist.example.com.", "example.invalid.")
1001
badresults = {}
1002
for iname in badnames:
1003
try:
1004
result = socket.getaddrinfo(iname, None, 0, 0,
1005
socket.SOCK_STREAM,
1006
socket.AI_CANONNAME)
1007
badresults[iname] = []
1008
for (_, _, _, cname, sockaddr) in result:
1009
badresults[iname].append("%s: %s" % (cname, sockaddr[0]))
1010
badips.add(sockaddr[0])
1011
except (socket.gaierror, socket.error):
1012
pass
1013
_DNS_REDIRECT_IP = badips
1014
if badresults:
1015
LOG.debug("detected dns redirection: %s", badresults)
1016
1017
try:
1018
result = socket.getaddrinfo(name, None)
1019
# check first result's sockaddr field
1020
addr = result[0][4][0]
1021
if addr in _DNS_REDIRECT_IP:
1022
LOG.debug("dns %s in _DNS_REDIRECT_IP", name)
1023
return False
1024
LOG.debug("dns %s resolved to '%s'", name, result)
1025
return True
1026
except (socket.gaierror, socket.error):
1027
LOG.debug("dns %s failed to resolve", name)
1028
return False
1029
1030
1031
def is_resolvable_url(url):
1032
"""determine if this url is resolvable (existing or ip)."""
1033
return is_resolvable(urlparse(url).hostname)
1034
1035
1036
def target_path(target, path=None):
1037
# return 'path' inside target, accepting target as None
1038
if target in (None, ""):
1039
target = "/"
1040
elif not isinstance(target, string_types):
1041
raise ValueError("Unexpected input for target: %s" % target)
1042
else:
1043
target = os.path.abspath(target)
1044
# abspath("//") returns "//" specifically for 2 slashes.
1045
if target.startswith("//"):
1046
target = target[1:]
1047
1048
if not path:
1049
return target
1050
1051
# os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /.
1052
while len(path) and path[0] == "/":
1053
path = path[1:]
1054
1055
return os.path.join(target, path)
1056
1057
898
1058
# vi: ts=4 expandtab syntax=python
Loggerhead is a web-based interface for Breezy
Version: 2.0.1