1
# Copyright 2014-2015 Canonical Limited.
3
# This file is part of charm-helpers.
5
# charm-helpers is free software: you can redistribute it and/or modify
6
# it under the terms of the GNU Lesser General Public License version 3 as
7
# published by the Free Software Foundation.
9
# charm-helpers is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU Lesser General Public License for more details.
14
# You should have received a copy of the GNU Lesser General Public License
15
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
18
from tempfile import NamedTemporaryFile
20
from yaml import safe_load
21
from charmhelpers.core.host import (
25
from charmhelpers.core.hookenv import (
33
from urllib.parse import urlparse, urlunparse
35
from urlparse import urlparse, urlunparse
38
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
39
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
41
PROPOSED_POCKET = """# Proposed
42
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
44
CLOUD_ARCHIVE_POCKETS = {
46
'folsom': 'precise-updates/folsom',
47
'precise-folsom': 'precise-updates/folsom',
48
'precise-folsom/updates': 'precise-updates/folsom',
49
'precise-updates/folsom': 'precise-updates/folsom',
50
'folsom/proposed': 'precise-proposed/folsom',
51
'precise-folsom/proposed': 'precise-proposed/folsom',
52
'precise-proposed/folsom': 'precise-proposed/folsom',
54
'grizzly': 'precise-updates/grizzly',
55
'precise-grizzly': 'precise-updates/grizzly',
56
'precise-grizzly/updates': 'precise-updates/grizzly',
57
'precise-updates/grizzly': 'precise-updates/grizzly',
58
'grizzly/proposed': 'precise-proposed/grizzly',
59
'precise-grizzly/proposed': 'precise-proposed/grizzly',
60
'precise-proposed/grizzly': 'precise-proposed/grizzly',
62
'havana': 'precise-updates/havana',
63
'precise-havana': 'precise-updates/havana',
64
'precise-havana/updates': 'precise-updates/havana',
65
'precise-updates/havana': 'precise-updates/havana',
66
'havana/proposed': 'precise-proposed/havana',
67
'precise-havana/proposed': 'precise-proposed/havana',
68
'precise-proposed/havana': 'precise-proposed/havana',
70
'icehouse': 'precise-updates/icehouse',
71
'precise-icehouse': 'precise-updates/icehouse',
72
'precise-icehouse/updates': 'precise-updates/icehouse',
73
'precise-updates/icehouse': 'precise-updates/icehouse',
74
'icehouse/proposed': 'precise-proposed/icehouse',
75
'precise-icehouse/proposed': 'precise-proposed/icehouse',
76
'precise-proposed/icehouse': 'precise-proposed/icehouse',
78
'juno': 'trusty-updates/juno',
79
'trusty-juno': 'trusty-updates/juno',
80
'trusty-juno/updates': 'trusty-updates/juno',
81
'trusty-updates/juno': 'trusty-updates/juno',
82
'juno/proposed': 'trusty-proposed/juno',
83
'trusty-juno/proposed': 'trusty-proposed/juno',
84
'trusty-proposed/juno': 'trusty-proposed/juno',
86
'kilo': 'trusty-updates/kilo',
87
'trusty-kilo': 'trusty-updates/kilo',
88
'trusty-kilo/updates': 'trusty-updates/kilo',
89
'trusty-updates/kilo': 'trusty-updates/kilo',
90
'kilo/proposed': 'trusty-proposed/kilo',
91
'trusty-kilo/proposed': 'trusty-proposed/kilo',
92
'trusty-proposed/kilo': 'trusty-proposed/kilo',
94
'liberty': 'trusty-updates/liberty',
95
'trusty-liberty': 'trusty-updates/liberty',
96
'trusty-liberty/updates': 'trusty-updates/liberty',
97
'trusty-updates/liberty': 'trusty-updates/liberty',
98
'liberty/proposed': 'trusty-proposed/liberty',
99
'trusty-liberty/proposed': 'trusty-proposed/liberty',
100
'trusty-proposed/liberty': 'trusty-proposed/liberty',
103
# The order of this list is very important. Handlers should be listed in from
104
# least- to most-specific URL matching.
106
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
107
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
108
'charmhelpers.fetch.giturl.GitUrlFetchHandler',
111
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
112
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
113
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
116
class SourceConfigError(Exception):
120
class UnhandledSource(Exception):
124
class AptLockError(Exception):
128
class BaseFetchHandler(object):
130
"""Base class for FetchHandler implementations in fetch plugins"""
132
def can_handle(self, source):
133
"""Returns True if the source can be handled. Otherwise returns
134
a string explaining why it cannot"""
135
return "Wrong source type"
137
def install(self, source):
138
"""Try to download and unpack the source. Return the path to the
139
unpacked files or raise UnhandledSource."""
140
raise UnhandledSource("Wrong source type {}".format(source))
142
def parse_url(self, url):
145
def base_url(self, url):
146
"""Return url without querystring or fragment"""
147
parts = list(self.parse_url(url))
148
parts[4:] = ['' for i in parts[4:]]
149
return urlunparse(parts)
152
def filter_installed_packages(packages):
153
"""Returns a list of packages that require installation"""
156
for package in packages:
159
p.current_ver or _pkgs.append(package)
161
log('Package {} has no installation candidate.'.format(package),
163
_pkgs.append(package)
167
def apt_cache(in_memory=True):
168
"""Build and return an apt cache"""
169
from apt import apt_pkg
172
apt_pkg.config.set("Dir::Cache::pkgcache", "")
173
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
174
return apt_pkg.Cache()
177
def apt_install(packages, options=None, fatal=False):
178
"""Install one or more packages"""
180
options = ['--option=Dpkg::Options::=--force-confold']
182
cmd = ['apt-get', '--assume-yes']
184
cmd.append('install')
185
if isinstance(packages, six.string_types):
189
log("Installing {} with options: {}".format(packages,
191
_run_apt_command(cmd, fatal)
194
def apt_upgrade(options=None, fatal=False, dist=False):
195
"""Upgrade all packages"""
197
options = ['--option=Dpkg::Options::=--force-confold']
199
cmd = ['apt-get', '--assume-yes']
202
cmd.append('dist-upgrade')
204
cmd.append('upgrade')
205
log("Upgrading with options: {}".format(options))
206
_run_apt_command(cmd, fatal)
209
def apt_update(fatal=False):
210
"""Update local apt cache"""
211
cmd = ['apt-get', 'update']
212
_run_apt_command(cmd, fatal)
215
def apt_purge(packages, fatal=False):
216
"""Purge one or more packages"""
217
cmd = ['apt-get', '--assume-yes', 'purge']
218
if isinstance(packages, six.string_types):
222
log("Purging {}".format(packages))
223
_run_apt_command(cmd, fatal)
226
def apt_mark(packages, mark, fatal=False):
227
"""Flag one or more packages using apt-mark"""
228
cmd = ['apt-mark', mark]
229
if isinstance(packages, six.string_types):
233
log("Holding {}".format(packages))
236
subprocess.check_call(cmd, universal_newlines=True)
238
subprocess.call(cmd, universal_newlines=True)
241
def apt_hold(packages, fatal=False):
242
return apt_mark(packages, 'hold', fatal=fatal)
245
def apt_unhold(packages, fatal=False):
246
return apt_mark(packages, 'unhold', fatal=fatal)
249
def add_source(source, key=None):
250
"""Add a package source to this system.
252
@param source: a URL or sources.list entry, as supported by
253
add-apt-repository(1). Examples::
256
deb https://stub:key@private.example.com/ubuntu trusty main
259
'proposed:' may be used to enable the standard 'proposed'
260
pocket for the release.
261
'cloud:' may be used to activate official cloud archive pockets,
262
such as 'cloud:icehouse'
263
'distro' may be used as a noop
265
@param key: A key to be added to the system's APT keyring and used
266
to verify the signatures on packages. Ideally, this should be an
267
ASCII format GPG public key including the block headers. A GPG key
268
id may also be used, but be aware that only insecure protocols are
269
available to retrieve the actual public key from a public keyserver
270
placing your Juju environment at risk. ppa and cloud archive keys
271
are securely added automtically, so sould not be provided.
274
log('Source is not present. Skipping')
277
if (source.startswith('ppa:') or
278
source.startswith('http') or
279
source.startswith('deb ') or
280
source.startswith('cloud-archive:')):
281
subprocess.check_call(['add-apt-repository', '--yes', source])
282
elif source.startswith('cloud:'):
283
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
285
pocket = source.split(':')[-1]
286
if pocket not in CLOUD_ARCHIVE_POCKETS:
287
raise SourceConfigError(
288
'Unsupported cloud: source option %s' %
290
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
291
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
292
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
293
elif source == 'proposed':
294
release = lsb_release()['DISTRIB_CODENAME']
295
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
296
apt.write(PROPOSED_POCKET.format(release))
297
elif source == 'distro':
300
log("Unknown source: {!r}".format(source))
303
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
304
with NamedTemporaryFile('w+') as key_file:
308
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
310
# Note that hkp: is in no way a secure protocol. Using a
311
# GPG key id is pointless from a security POV unless you
312
# absolutely trust your network and DNS.
313
subprocess.check_call(['apt-key', 'adv', '--keyserver',
314
'hkp://keyserver.ubuntu.com:80', '--recv',
318
def configure_sources(update=False,
319
sources_var='install_sources',
320
keys_var='install_keys'):
322
Configure multiple sources from charm configuration.
324
The lists are encoded as yaml fragments in the configuration.
325
The frament needs to be included as a string. Sources and their
326
corresponding keys are of the types supported by add_source().
331
- "http://example.com/repo precise main"
336
Note that 'null' (a.k.a. None) should not be quoted.
338
sources = safe_load((config(sources_var) or '').strip()) or []
339
keys = safe_load((config(keys_var) or '').strip()) or None
341
if isinstance(sources, six.string_types):
345
for source in sources:
346
add_source(source, None)
348
if isinstance(keys, six.string_types):
351
if len(sources) != len(keys):
352
raise SourceConfigError(
353
'Install sources and keys lists are different lengths')
354
for source, key in zip(sources, keys):
355
add_source(source, key)
357
apt_update(fatal=True)
360
def install_remote(source, *args, **kwargs):
362
Install a file tree from a remote source
364
The specified source should be a url of the form:
365
scheme://[host]/path[#[option=value][&...]]
367
Schemes supported are based on this modules submodules.
368
Options supported are submodule-specific.
369
Additional arguments are passed through to the submodule.
373
dest = install_remote('http://example.com/archive.tgz',
377
This will download `archive.tgz`, validate it using SHA1 and, if
378
the file is ok, extract it and return the directory in which it
379
was extracted. If the checksum fails, it will raise
380
:class:`charmhelpers.core.host.ChecksumError`.
382
# We ONLY check for True here because can_handle may return a string
383
# explaining why it can't handle a given source.
384
handlers = [h for h in plugins() if h.can_handle(source) is True]
386
for handler in handlers:
388
installed_to = handler.install(source, *args, **kwargs)
389
except UnhandledSource as e:
390
log('Install source attempt unsuccessful: {}'.format(e),
393
raise UnhandledSource("No handler found for source {}".format(source))
397
def install_from_config(config_var_name):
398
charm_config = config()
399
source = charm_config[config_var_name]
400
return install_remote(source)
403
def plugins(fetch_handlers=None):
404
if not fetch_handlers:
405
fetch_handlers = FETCH_HANDLERS
407
for handler_name in fetch_handlers:
408
package, classname = handler_name.rsplit('.', 1)
410
handler_class = getattr(
411
importlib.import_module(package),
413
plugin_list.append(handler_class())
414
except (ImportError, AttributeError):
415
# Skip missing plugins so that they can be ommitted from
416
# installation if desired
417
log("FetchHandler {} not found, skipping plugin".format(
422
def _run_apt_command(cmd, fatal=False):
424
Run an APT command, checking output and retrying if the fatal flag is set
427
:param: cmd: str: The apt command to run.
428
:param: fatal: bool: Whether the command's output should be checked and
431
env = os.environ.copy()
433
if 'DEBIAN_FRONTEND' not in env:
434
env['DEBIAN_FRONTEND'] = 'noninteractive'
440
# If the command is considered "fatal", we need to retry if the apt
441
# lock was not acquired.
443
while result is None or result == APT_NO_LOCK:
445
result = subprocess.check_call(cmd, env=env)
446
except subprocess.CalledProcessError as e:
447
retry_count = retry_count + 1
448
if retry_count > APT_NO_LOCK_RETRY_COUNT:
450
result = e.returncode
451
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
452
"".format(APT_NO_LOCK_RETRY_DELAY))
453
time.sleep(APT_NO_LOCK_RETRY_DELAY)
456
subprocess.call(cmd, env=env)
added
removed
charmhelpers/fetch/__init__.py
