3
# Copyright (C) 2009-2010 Canonical Ltd.
4
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
6
# Author: Scott Moser <scott.moser@canonical.com>
7
# Author: Juerg Hafliger <juerg.haefliger@hp.com>
9
# This program is free software: you can redistribute it and/or modify
10
# it under the terms of the GNU General Public License version 3, as
11
# published by the Free Software Foundation.
13
# This program is distributed in the hope that it will be useful,
14
# but WITHOUT ANY WARRANTY; without even the implied warranty of
15
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
# GNU General Public License for more details.
18
# You should have received a copy of the GNU General Public License
19
# along with this program. If not, see <http://www.gnu.org/licenses/>.
27
from Cheetah.Template import Template
41
HAVE_LIBSELINUX = True
43
HAVE_LIBSELINUX = False
48
stream = open(fname, "r")
49
conf = yaml.load(stream)
53
if e.errno == errno.ENOENT:
58
def get_base_cfg(cfgfile, cfg_builtin="", parsed_cfgs=None):
61
if parsed_cfgs and cfgfile in parsed_cfgs:
62
return(parsed_cfgs[cfgfile])
64
syscfg = read_conf_with_confd(cfgfile)
66
kern_contents = read_cc_from_cmdline()
68
kerncfg = yaml.load(kern_contents)
70
# kernel parameters override system config
71
combined = mergedict(kerncfg, syscfg)
74
builtin = yaml.load(cfg_builtin)
75
fin = mergedict(combined, builtin)
79
if parsed_cfgs != None:
80
parsed_cfgs[cfgfile] = fin
84
def get_cfg_option_bool(yobj, key, default=False):
90
if str(val).lower() in ['true', '1', 'on', 'yes']:
95
def get_cfg_option_str(yobj, key, default=None):
101
def get_cfg_option_list_or_str(yobj, key, default=None):
103
Gets the C{key} config option from C{yobj} as a list of strings. If the
104
key is present as a single string it will be returned as a list with one
107
@param yobj: The configuration object.
108
@param key: The configuration key to get.
109
@param default: The default to return if key is not found.
110
@return: The configuration option as a list of strings or default if key
115
if yobj[key] is None:
117
if isinstance(yobj[key], list):
122
# get a cfg entry by its path array
123
# for f['a']['b']: get_cfg_by_path(mycfg,('a','b'))
124
def get_cfg_by_path(yobj, keyp, default=None):
133
def mergedict(src, cand):
135
Merge values from C{cand} into C{src}. If C{src} has a key C{cand} will
136
not override. Nested dictionaries are merged recursively.
138
if isinstance(src, dict) and isinstance(cand, dict):
139
for k, v in cand.iteritems():
143
src[k] = mergedict(src[k], v)
147
def delete_dir_contents(dirname):
149
Deletes all contents of a directory without deleting the directory itself.
151
@param dirname: The directory whose contents should be deleted.
153
for node in os.listdir(dirname):
154
node_fullpath = os.path.join(dirname, node)
155
if os.path.isdir(node_fullpath):
156
shutil.rmtree(node_fullpath)
158
os.unlink(node_fullpath)
161
def write_file(filename, content, mode=0644, omode="wb"):
163
Writes a file with the given content and sets the file mode as specified.
164
Resotres the SELinux context if possible.
166
@param filename: The full path of the file to write.
167
@param content: The content to write to the file.
168
@param mode: The filesystem mode to set on the file.
169
@param omode: The open mode used when opening the file (r, rb, a, etc.)
172
os.makedirs(os.path.dirname(filename))
174
if e.errno != errno.EEXIST:
177
f = open(filename, omode)
179
os.chmod(filename, mode)
182
restorecon_if_possible(filename)
185
def restorecon_if_possible(path, recursive=False):
186
if HAVE_LIBSELINUX and selinux.is_selinux_enabled():
187
selinux.restorecon(path, recursive=recursive)
190
# get keyid from keyserver
191
def getkeybyid(keyid, keyserver):
195
[ -n "$k" ] || exit 1;
196
armour=$(gpg --list-keys --armour "${k}")
197
if [ -z "${armour}" ]; then
198
gpg --keyserver ${ks} --recv $k >/dev/null &&
199
armour=$(gpg --export --armour "${k}") &&
200
gpg --batch --yes --delete-keys "${k}"
202
[ -n "${armour}" ] && echo "${armour}"
204
args = ['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver]
205
return(subp(args)[0])
208
def runparts(dirp, skip_no_exist=True):
209
if skip_no_exist and not os.path.isdir(dirp):
213
for exe_name in sorted(os.listdir(dirp)):
214
exe_path = os.path.join(dirp, exe_name)
215
if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK):
216
popen = subprocess.Popen([exe_path])
218
if popen.returncode is not 0:
220
sys.stderr.write("failed: %s [%i]\n" %
221
(exe_path, popen.returncode))
223
raise RuntimeError('runparts: %i failures' % failed)
226
def subp(args, input_=None):
227
sp = subprocess.Popen(args, stdout=subprocess.PIPE,
228
stderr=subprocess.PIPE, stdin=subprocess.PIPE)
229
out, err = sp.communicate(input_)
230
if sp.returncode is not 0:
231
raise subprocess.CalledProcessError(sp.returncode, args, (out, err))
235
def render_to_file(template, outfile, searchList):
236
t = Template(file='/etc/cloud/templates/%s.tmpl' % template,
237
searchList=[searchList])
238
f = open(outfile, 'w')
243
def render_string(template, searchList):
244
return(Template(template, searchList=[searchList]).respond())
248
# returns boolean indicating success or failure (presense of files)
249
# if files are present, populates 'fill' dictionary with 'user-data' and
250
# 'meta-data' entries
251
def read_optional_seed(fill, base="", ext="", timeout=5):
253
(md, ud) = read_seeded(base, ext, timeout)
254
fill['user-data'] = ud
255
fill['meta-data'] = md
258
if e.errno == errno.ENOENT:
263
# raise OSError with enoent if not found
264
def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
265
if base.startswith("/"):
266
base = "file://%s" % base
268
# default retries for file is 0. for network is 10
269
if base.startswith("file://"):
270
retries = file_retries
272
if base.find("%s") >= 0:
273
ud_url = base % ("user-data" + ext)
274
md_url = base % ("meta-data" + ext)
276
ud_url = "%s%s%s" % (base, "user-data", ext)
277
md_url = "%s%s%s" % (base, "meta-data", ext)
281
for attempt in range(0, retries + 1):
283
md_str = readurl(md_url, timeout=timeout)
284
ud = readurl(ud_url, timeout=timeout)
285
md = yaml.load(md_str)
288
except urllib2.HTTPError as e:
290
except urllib2.URLError as e:
292
if (isinstance(e.reason, OSError) and
293
e.reason.errno == errno.ENOENT):
296
if attempt == retries:
299
#print "%s failed, sleeping" % attempt
305
def logexc(log, lvl=logging.DEBUG):
306
log.log(lvl, traceback.format_exc())
309
class RecursiveInclude(Exception):
313
def read_file_with_includes(fname, rel=".", stack=None, patt=None):
316
if not fname.startswith("/"):
317
fname = os.sep.join((rel, fname))
319
fname = os.path.realpath(fname)
322
raise(RecursiveInclude("%s recursively included" % fname))
324
raise(RecursiveInclude("%s included, stack size = %i" %
325
(fname, len(stack))))
328
patt = re.compile("^#(opt_include|include)[ \t].*$", re.MULTILINE)
337
rel = os.path.dirname(fname)
342
match = patt.search(contents[cur:])
345
loc = match.start() + cur
346
endl = match.end() + cur
348
(key, cur_fname) = contents[loc:endl].split(None, 2)
349
cur_fname = cur_fname.strip()
352
inc_contents = read_file_with_includes(cur_fname, rel, stack, patt)
354
if e.errno == errno.ENOENT and key == "#opt_include":
358
contents = contents[0:loc] + inc_contents + contents[endl + 1:]
359
cur = loc + len(inc_contents)
364
def read_conf_d(confd):
365
# get reverse sorted list (later trumps newer)
366
confs = sorted(os.listdir(confd), reverse=True)
368
# remove anything not ending in '.cfg'
369
confs = [f for f in confs if f.endswith(".cfg")]
371
# remove anything not a file
372
confs = [f for f in confs if os.path.isfile("%s/%s" % (confd, f))]
376
cfg = mergedict(cfg, read_conf("%s/%s" % (confd, conf)))
381
def read_conf_with_confd(cfgfile):
382
cfg = read_conf(cfgfile)
385
if cfg['conf_d'] is not None:
386
confd = cfg['conf_d']
387
if not isinstance(confd, str):
388
raise Exception("cfgfile %s contains 'conf_d' "
389
"with non-string" % cfgfile)
390
elif os.path.isdir("%s.d" % cfgfile):
391
confd = "%s.d" % cfgfile
396
confd_cfg = read_conf_d(confd)
398
return(mergedict(confd_cfg, cfg))
402
if 'DEBUG_PROC_CMDLINE' in os.environ:
403
cmdline = os.environ["DEBUG_PROC_CMDLINE"]
406
cmdfp = open("/proc/cmdline")
407
cmdline = cmdfp.read().strip()
414
def read_cc_from_cmdline(cmdline=None):
415
# this should support reading cloud-config information from
416
# the kernel command line. It is intended to support content of the
418
# cc: <yaml content here> [end_cc]
419
# this would include:
420
# cc: ssh_import_id: [smoser, kirkland]\\n
421
# cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc
422
# cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc
424
cmdline = get_cmdline()
428
begin_l = len(tag_begin)
432
begin = cmdline.find(tag_begin)
434
end = cmdline.find(tag_end, begin + begin_l)
437
tokens.append(cmdline[begin + begin_l:end].lstrip().replace("\\n",
440
begin = cmdline.find(tag_begin, end + end_l)
442
return('\n'.join(tokens))
445
def ensure_dirs(dirlist, mode=0755):
454
if e.errno != errno.EEXIST:
463
def chownbyname(fname, user=None, group=None):
466
if user == None and group == None:
470
uid = pwd.getpwnam(user).pw_uid
473
gid = grp.getgrnam(group).gr_gid
475
os.chown(fname, uid, gid)
478
def readurl(url, data=None, timeout=None):
481
openargs['timeout'] = timeout
484
req = urllib2.Request(url)
486
encoded = urllib.urlencode(data)
487
req = urllib2.Request(url, encoded)
489
response = urllib2.urlopen(req, **openargs)
490
return(response.read())
493
# shellify, takes a list of commands
494
# for each entry in the list
495
# if it is an array, shell protect it (with single ticks)
496
# if it is a string, do nothing
497
def shellify(cmdlist):
498
content = "#!/bin/sh\n"
499
escaped = "%s%s%s%s" % ("'", '\\', "'", "'")
501
# if the item is a list, wrap all items in single tick
502
# if its not, then just write it directly
503
if isinstance(args, list):
506
fixed.append("'%s'" % str(f).replace("'", escaped))
507
content = "%s%s\n" % (content, ' '.join(fixed))
509
content = "%s%s\n" % (content, str(args))
513
def dos2unix(string):
514
# find first end of line
515
pos = string.find('\n')
516
if pos <= 0 or string[pos - 1] != '\r':
518
return(string.replace('\r\n', '\n'))
522
# is this code running in a container of some sort
524
for helper in ('running-in-container', 'lxc-is-container'):
526
# try to run a helper program. if it returns true
527
# then we're inside a container. otherwise, no
528
sp = subprocess.Popen(helper, stdout=subprocess.PIPE,
529
stderr=subprocess.PIPE)
531
return(sp.returncode == 0)
533
if e.errno != errno.ENOENT:
536
# this code is largely from the logic in
537
# ubuntu's /etc/init/container-detect.conf
539
# Detect old-style libvirt
540
# Detect OpenVZ containers
541
pid1env = get_proc_env(1)
542
if "container" in pid1env:
545
if "LIBVIRT_LXC_UUID" in pid1env:
549
if e.errno != errno.ENOENT:
552
# Detect OpenVZ containers
553
if os.path.isdir("/proc/vz") and not os.path.isdir("/proc/bc"):
557
# Detect Vserver containers
558
with open("/proc/self/status") as fp:
559
lines = fp.read().splitlines()
561
if line.startswith("VxID:"):
562
(_key, val) = line.strip().split(":", 1)
566
if e.errno != errno.ENOENT:
572
def get_proc_env(pid):
573
# return the environment in a dict that a given process id was started with
575
with open("/proc/%s/environ" % pid) as fp:
576
toks = fp.read().split("\0")
580
(name, val) = tok.split("=", 1)
585
def get_hostname_fqdn(cfg, cloud):
586
# return the hostname and fqdn from 'cfg'. If not found in cfg,
587
# then fall back to data from cloud
589
# user specified a fqdn. Default hostname then is based off that
591
hostname = get_cfg_option_str(cfg, "hostname", fqdn.split('.')[0])
593
if "hostname" in cfg and cfg['hostname'].find('.') > 0:
594
# user specified hostname, and it had '.' in it
595
# be nice to them. set fqdn and hostname from that
596
fqdn = cfg['hostname']
597
hostname = cfg['hostname'][:fqdn.find('.')]
599
# no fqdn set, get fqdn from cloud.
600
# get hostname from cfg if available otherwise cloud
601
fqdn = cloud.get_hostname(fqdn=True)
602
if "hostname" in cfg:
603
hostname = cfg['hostname']
605
hostname = cloud.get_hostname()
606
return(hostname, fqdn)
609
def get_fqdn_from_hosts(hostname, filename="/etc/hosts"):
610
# this parses /etc/hosts to get a fqdn. It should return the same
611
# result as 'hostname -f <hostname>' if /etc/hosts.conf
612
# did not have did not have 'bind' in the order attribute
615
with open(filename, "r") as hfp:
616
for line in hfp.readlines():
617
hashpos = line.find("#")
619
line = line[0:hashpos]
622
# if there there is less than 3 entries (ip, canonical, alias)
623
# then ignore this line
627
if hostname in toks[2:]:
632
if e.errno == errno.ENOENT:
638
def is_resolvable(name):
639
""" determine if a url is resolvable, return a boolean """
641
socket.getaddrinfo(name, None)
643
except socket.gaierror:
647
def is_resolvable_url(url):
648
""" determine if this url is resolvable (existing or ip) """
649
return(is_resolvable(urlparse.urlparse(url).hostname))
652
def search_for_mirror(candidates):
653
""" Search through a list of mirror urls for one that works """
654
for cand in candidates:
656
if is_resolvable_url(cand):
666
reopen stdin as /dev/null so even subprocesses or other os level things get
669
if _CLOUD_INIT_SAVE_STDIN is set in environment to a non empty or '0' value
670
then input will not be closed (only useful potentially for debugging).
672
if os.environ.get("_CLOUD_INIT_SAVE_STDIN") in ("", "0", False):
674
with open(os.devnull) as fp:
675
os.dup2(fp.fileno(), sys.stdin.fileno())
678
def find_devs_with(criteria):
680
find devices matching given criteria (via blkid)
681
criteria can be *one* of:
687
(out, _err) = subp(['blkid', '-t%s' % criteria, '-odevice'])
688
except subprocess.CalledProcessError:
690
return(str(out).splitlines())
693
class mountFailedError(Exception):
697
def mount_callback_umount(device, callback, data=None):
699
mount the device, call method 'callback' passing the directory
700
in which it was mounted, then unmount. Return whatever 'callback'
701
returned. If data != None, also pass data to callback.
704
def _cleanup(umount, tmpd):
707
subp(["umount", '-l', umount])
708
except subprocess.CalledProcessError:
713
# go through mounts to see if it was already mounted
714
fp = open("/proc/mounts")
715
mounts = fp.readlines()
721
for mpline in mounts:
722
(dev, mp, fstype, _opts, _freq, _passno) = mpline.split()
723
mp = mp.replace("\\040", " ")
724
mounted[dev] = (dev, fstype, mp, False)
727
if device in mounted:
728
mountpoint = "%s/" % mounted[device][2]
730
tmpd = tempfile.mkdtemp()
732
mountcmd = ["mount", "-o", "ro", device, tmpd]
735
(_out, _err) = subp(mountcmd)
737
except subprocess.CalledProcessError as exc:
738
_cleanup(umount, tmpd)
739
raise mountFailedError(exc.output[1])
741
mountpoint = "%s/" % tmpd
745
ret = callback(mountpoint)
747
ret = callback(mountpoint, data)
749
except Exception as exc:
750
_cleanup(umount, tmpd)
753
_cleanup(umount, tmpd)
758
def wait_for_url(urls, max_wait=None, timeout=None,
759
status_cb=None, headers_cb=None, exception_cb=None):
761
urls: a list of urls to try
762
max_wait: roughly the maximum time to wait before giving up
763
The max time is *actually* len(urls)*timeout as each url will
764
be tried once and given the timeout provided.
765
timeout: the timeout provided to urllib2.urlopen
766
status_cb: call method with string message when a url is not available
767
headers_cb: call method with single argument of url to get headers
769
exception_cb: call method with 2 arguments 'msg' (per status_cb) and
770
'exception', the exception that occurred.
772
the idea of this routine is to wait for the EC2 metdata service to
773
come up. On both Eucalyptus and EC2 we have seen the case where
774
the instance hit the MD before the MD service was up. EC2 seems
775
to have permenantely fixed this, though.
777
In openstack, the metadata service might be painfully slow, and
778
unable to avoid hitting a timeout of even up to 10 seconds or more
779
(LP: #894279) for a simple GET.
781
Offset those needs with the need to not hang forever (and block boot)
782
on a system where cloud-init is configured to look for EC2 Metadata
783
service but is not going to find one. It is possible that the instance
784
data host (169.254.169.254) may be firewalled off Entirely for a sytem,
785
meaning that the connection will block forever unless a timeout is set.
787
starttime = time.time()
791
def nullstatus_cb(msg):
794
if status_cb == None:
795
status_cb = nullstatus_cb
797
def timeup(max_wait, starttime):
798
return((max_wait <= 0 or max_wait == None) or
799
(time.time() - starttime > max_wait))
803
sleeptime = int(loop_n / 5) + 1
807
if timeup(max_wait, starttime):
809
if timeout and (now + timeout > (starttime + max_wait)):
810
# shorten timeout to not run way over max_time
811
timeout = int((starttime + max_wait) - now)
815
if headers_cb != None:
816
headers = headers_cb(url)
820
req = urllib2.Request(url, data=None, headers=headers)
821
resp = urllib2.urlopen(req, timeout=timeout)
822
contents = resp.read()
824
reason = "empty data [%s]" % (resp.code)
825
e = ValueError(reason)
826
elif not (resp.code >= 200 and resp.code < 400):
827
reason = "bad status code [%s]" % (resp.code)
828
e = ValueError(reason)
831
except urllib2.HTTPError as e:
832
reason = "http error [%s]" % e.code
833
except urllib2.URLError as e:
834
reason = "url error [%s]" % e.reason
835
except socket.timeout as e:
836
reason = "socket timeout [%s]" % e
837
except Exception as e:
838
reason = "unexpected error [%s]" % e
840
status_msg = ("'%s' failed [%s/%ss]: %s" %
841
(url, int(time.time() - starttime), max_wait,
843
status_cb(status_msg)
845
exception_cb(msg=status_msg, exception=e)
847
if timeup(max_wait, starttime):
851
time.sleep(sleeptime)
856
def keyval_str_to_dict(kvstring):
858
for tok in kvstring.split():
860
(key, val) = tok.split("=", 1)
added
removed
.pc/lp-974509-detect-dns-server-redirection.patch/cloudinit/util.py
