1
# Copyright 2014-2015 Canonical Limited.
3
# This file is part of charm-helpers.
5
# charm-helpers is free software: you can redistribute it and/or modify
6
# it under the terms of the GNU Lesser General Public License version 3 as
7
# published by the Free Software Foundation.
9
# charm-helpers is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU Lesser General Public License for more details.
14
# You should have received a copy of the GNU Lesser General Public License
15
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
18
from tempfile import NamedTemporaryFile
20
from yaml import safe_load
21
from charmhelpers.core.host import (
25
from charmhelpers.core.hookenv import (
33
from urllib.parse import urlparse, urlunparse
35
from urlparse import urlparse, urlunparse
38
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
39
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
41
PROPOSED_POCKET = """# Proposed
42
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
44
CLOUD_ARCHIVE_POCKETS = {
46
'folsom': 'precise-updates/folsom',
47
'precise-folsom': 'precise-updates/folsom',
48
'precise-folsom/updates': 'precise-updates/folsom',
49
'precise-updates/folsom': 'precise-updates/folsom',
50
'folsom/proposed': 'precise-proposed/folsom',
51
'precise-folsom/proposed': 'precise-proposed/folsom',
52
'precise-proposed/folsom': 'precise-proposed/folsom',
54
'grizzly': 'precise-updates/grizzly',
55
'precise-grizzly': 'precise-updates/grizzly',
56
'precise-grizzly/updates': 'precise-updates/grizzly',
57
'precise-updates/grizzly': 'precise-updates/grizzly',
58
'grizzly/proposed': 'precise-proposed/grizzly',
59
'precise-grizzly/proposed': 'precise-proposed/grizzly',
60
'precise-proposed/grizzly': 'precise-proposed/grizzly',
62
'havana': 'precise-updates/havana',
63
'precise-havana': 'precise-updates/havana',
64
'precise-havana/updates': 'precise-updates/havana',
65
'precise-updates/havana': 'precise-updates/havana',
66
'havana/proposed': 'precise-proposed/havana',
67
'precise-havana/proposed': 'precise-proposed/havana',
68
'precise-proposed/havana': 'precise-proposed/havana',
70
'icehouse': 'precise-updates/icehouse',
71
'precise-icehouse': 'precise-updates/icehouse',
72
'precise-icehouse/updates': 'precise-updates/icehouse',
73
'precise-updates/icehouse': 'precise-updates/icehouse',
74
'icehouse/proposed': 'precise-proposed/icehouse',
75
'precise-icehouse/proposed': 'precise-proposed/icehouse',
76
'precise-proposed/icehouse': 'precise-proposed/icehouse',
78
'juno': 'trusty-updates/juno',
79
'trusty-juno': 'trusty-updates/juno',
80
'trusty-juno/updates': 'trusty-updates/juno',
81
'trusty-updates/juno': 'trusty-updates/juno',
82
'juno/proposed': 'trusty-proposed/juno',
83
'trusty-juno/proposed': 'trusty-proposed/juno',
84
'trusty-proposed/juno': 'trusty-proposed/juno',
86
'kilo': 'trusty-updates/kilo',
87
'trusty-kilo': 'trusty-updates/kilo',
88
'trusty-kilo/updates': 'trusty-updates/kilo',
89
'trusty-updates/kilo': 'trusty-updates/kilo',
90
'kilo/proposed': 'trusty-proposed/kilo',
91
'trusty-kilo/proposed': 'trusty-proposed/kilo',
92
'trusty-proposed/kilo': 'trusty-proposed/kilo',
95
# The order of this list is very important. Handlers should be listed in from
96
# least- to most-specific URL matching.
98
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
99
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
100
'charmhelpers.fetch.giturl.GitUrlFetchHandler',
103
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
104
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
105
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
108
class SourceConfigError(Exception):
112
class UnhandledSource(Exception):
116
class AptLockError(Exception):
120
class BaseFetchHandler(object):
122
"""Base class for FetchHandler implementations in fetch plugins"""
124
def can_handle(self, source):
125
"""Returns True if the source can be handled. Otherwise returns
126
a string explaining why it cannot"""
127
return "Wrong source type"
129
def install(self, source):
130
"""Try to download and unpack the source. Return the path to the
131
unpacked files or raise UnhandledSource."""
132
raise UnhandledSource("Wrong source type {}".format(source))
134
def parse_url(self, url):
137
def base_url(self, url):
138
"""Return url without querystring or fragment"""
139
parts = list(self.parse_url(url))
140
parts[4:] = ['' for i in parts[4:]]
141
return urlunparse(parts)
144
def filter_installed_packages(packages):
145
"""Returns a list of packages that require installation"""
148
for package in packages:
151
p.current_ver or _pkgs.append(package)
153
log('Package {} has no installation candidate.'.format(package),
155
_pkgs.append(package)
159
def apt_cache(in_memory=True):
160
"""Build and return an apt cache"""
164
apt_pkg.config.set("Dir::Cache::pkgcache", "")
165
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
166
return apt_pkg.Cache()
169
def apt_install(packages, options=None, fatal=False):
170
"""Install one or more packages"""
172
options = ['--option=Dpkg::Options::=--force-confold']
174
cmd = ['apt-get', '--assume-yes']
176
cmd.append('install')
177
if isinstance(packages, six.string_types):
181
log("Installing {} with options: {}".format(packages,
183
_run_apt_command(cmd, fatal)
186
def apt_upgrade(options=None, fatal=False, dist=False):
187
"""Upgrade all packages"""
189
options = ['--option=Dpkg::Options::=--force-confold']
191
cmd = ['apt-get', '--assume-yes']
194
cmd.append('dist-upgrade')
196
cmd.append('upgrade')
197
log("Upgrading with options: {}".format(options))
198
_run_apt_command(cmd, fatal)
201
def apt_update(fatal=False):
202
"""Update local apt cache"""
203
cmd = ['apt-get', 'update']
204
_run_apt_command(cmd, fatal)
207
def apt_purge(packages, fatal=False):
208
"""Purge one or more packages"""
209
cmd = ['apt-get', '--assume-yes', 'purge']
210
if isinstance(packages, six.string_types):
214
log("Purging {}".format(packages))
215
_run_apt_command(cmd, fatal)
218
def apt_hold(packages, fatal=False):
219
"""Hold one or more packages"""
220
cmd = ['apt-mark', 'hold']
221
if isinstance(packages, six.string_types):
225
log("Holding {}".format(packages))
228
subprocess.check_call(cmd)
233
def add_source(source, key=None):
234
"""Add a package source to this system.
236
@param source: a URL or sources.list entry, as supported by
237
add-apt-repository(1). Examples::
240
deb https://stub:key@private.example.com/ubuntu trusty main
243
'proposed:' may be used to enable the standard 'proposed'
244
pocket for the release.
245
'cloud:' may be used to activate official cloud archive pockets,
246
such as 'cloud:icehouse'
247
'distro' may be used as a noop
249
@param key: A key to be added to the system's APT keyring and used
250
to verify the signatures on packages. Ideally, this should be an
251
ASCII format GPG public key including the block headers. A GPG key
252
id may also be used, but be aware that only insecure protocols are
253
available to retrieve the actual public key from a public keyserver
254
placing your Juju environment at risk. ppa and cloud archive keys
255
are securely added automtically, so sould not be provided.
258
log('Source is not present. Skipping')
261
if (source.startswith('ppa:') or
262
source.startswith('http') or
263
source.startswith('deb ') or
264
source.startswith('cloud-archive:')):
265
subprocess.check_call(['add-apt-repository', '--yes', source])
266
elif source.startswith('cloud:'):
267
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
269
pocket = source.split(':')[-1]
270
if pocket not in CLOUD_ARCHIVE_POCKETS:
271
raise SourceConfigError(
272
'Unsupported cloud: source option %s' %
274
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
275
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
276
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
277
elif source == 'proposed':
278
release = lsb_release()['DISTRIB_CODENAME']
279
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
280
apt.write(PROPOSED_POCKET.format(release))
281
elif source == 'distro':
284
log("Unknown source: {!r}".format(source))
287
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
288
with NamedTemporaryFile('w+') as key_file:
292
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
294
# Note that hkp: is in no way a secure protocol. Using a
295
# GPG key id is pointless from a security POV unless you
296
# absolutely trust your network and DNS.
297
subprocess.check_call(['apt-key', 'adv', '--keyserver',
298
'hkp://keyserver.ubuntu.com:80', '--recv',
302
def configure_sources(update=False,
303
sources_var='install_sources',
304
keys_var='install_keys'):
306
Configure multiple sources from charm configuration.
308
The lists are encoded as yaml fragments in the configuration.
309
The frament needs to be included as a string. Sources and their
310
corresponding keys are of the types supported by add_source().
315
- "http://example.com/repo precise main"
320
Note that 'null' (a.k.a. None) should not be quoted.
322
sources = safe_load((config(sources_var) or '').strip()) or []
323
keys = safe_load((config(keys_var) or '').strip()) or None
325
if isinstance(sources, six.string_types):
329
for source in sources:
330
add_source(source, None)
332
if isinstance(keys, six.string_types):
335
if len(sources) != len(keys):
336
raise SourceConfigError(
337
'Install sources and keys lists are different lengths')
338
for source, key in zip(sources, keys):
339
add_source(source, key)
341
apt_update(fatal=True)
344
def install_remote(source, *args, **kwargs):
346
Install a file tree from a remote source
348
The specified source should be a url of the form:
349
scheme://[host]/path[#[option=value][&...]]
351
Schemes supported are based on this modules submodules.
352
Options supported are submodule-specific.
353
Additional arguments are passed through to the submodule.
357
dest = install_remote('http://example.com/archive.tgz',
361
This will download `archive.tgz`, validate it using SHA1 and, if
362
the file is ok, extract it and return the directory in which it
363
was extracted. If the checksum fails, it will raise
364
:class:`charmhelpers.core.host.ChecksumError`.
366
# We ONLY check for True here because can_handle may return a string
367
# explaining why it can't handle a given source.
368
handlers = [h for h in plugins() if h.can_handle(source) is True]
370
for handler in handlers:
372
installed_to = handler.install(source, *args, **kwargs)
373
except UnhandledSource:
376
raise UnhandledSource("No handler found for source {}".format(source))
380
def install_from_config(config_var_name):
381
charm_config = config()
382
source = charm_config[config_var_name]
383
return install_remote(source)
386
def plugins(fetch_handlers=None):
387
if not fetch_handlers:
388
fetch_handlers = FETCH_HANDLERS
390
for handler_name in fetch_handlers:
391
package, classname = handler_name.rsplit('.', 1)
393
handler_class = getattr(
394
importlib.import_module(package),
396
plugin_list.append(handler_class())
397
except (ImportError, AttributeError):
398
# Skip missing plugins so that they can be ommitted from
399
# installation if desired
400
log("FetchHandler {} not found, skipping plugin".format(
405
def _run_apt_command(cmd, fatal=False):
407
Run an APT command, checking output and retrying if the fatal flag is set
410
:param: cmd: str: The apt command to run.
411
:param: fatal: bool: Whether the command's output should be checked and
414
env = os.environ.copy()
416
if 'DEBIAN_FRONTEND' not in env:
417
env['DEBIAN_FRONTEND'] = 'noninteractive'
423
# If the command is considered "fatal", we need to retry if the apt
424
# lock was not acquired.
426
while result is None or result == APT_NO_LOCK:
428
result = subprocess.check_call(cmd, env=env)
429
except subprocess.CalledProcessError as e:
430
retry_count = retry_count + 1
431
if retry_count > APT_NO_LOCK_RETRY_COUNT:
433
result = e.returncode
434
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
435
"".format(APT_NO_LOCK_RETRY_DELAY))
436
time.sleep(APT_NO_LOCK_RETRY_DELAY)
439
subprocess.call(cmd, env=env)
added
removed
hooks/charmhelpers/fetch/__init__.py
