1
# Copyright 2014-2015 Canonical Limited.
3
# This file is part of charm-helpers.
5
# charm-helpers is free software: you can redistribute it and/or modify
6
# it under the terms of the GNU Lesser General Public License version 3 as
7
# published by the Free Software Foundation.
9
# charm-helpers is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU Lesser General Public License for more details.
14
# You should have received a copy of the GNU Lesser General Public License
15
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
18
from tempfile import NamedTemporaryFile
20
from yaml import safe_load
21
from charmhelpers.core.host import (
25
from charmhelpers.core.hookenv import (
33
from urllib.parse import urlparse, urlunparse
35
from urlparse import urlparse, urlunparse
38
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
39
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
41
PROPOSED_POCKET = """# Proposed
42
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
44
CLOUD_ARCHIVE_POCKETS = {
46
'folsom': 'precise-updates/folsom',
47
'precise-folsom': 'precise-updates/folsom',
48
'precise-folsom/updates': 'precise-updates/folsom',
49
'precise-updates/folsom': 'precise-updates/folsom',
50
'folsom/proposed': 'precise-proposed/folsom',
51
'precise-folsom/proposed': 'precise-proposed/folsom',
52
'precise-proposed/folsom': 'precise-proposed/folsom',
54
'grizzly': 'precise-updates/grizzly',
55
'precise-grizzly': 'precise-updates/grizzly',
56
'precise-grizzly/updates': 'precise-updates/grizzly',
57
'precise-updates/grizzly': 'precise-updates/grizzly',
58
'grizzly/proposed': 'precise-proposed/grizzly',
59
'precise-grizzly/proposed': 'precise-proposed/grizzly',
60
'precise-proposed/grizzly': 'precise-proposed/grizzly',
62
'havana': 'precise-updates/havana',
63
'precise-havana': 'precise-updates/havana',
64
'precise-havana/updates': 'precise-updates/havana',
65
'precise-updates/havana': 'precise-updates/havana',
66
'havana/proposed': 'precise-proposed/havana',
67
'precise-havana/proposed': 'precise-proposed/havana',
68
'precise-proposed/havana': 'precise-proposed/havana',
70
'icehouse': 'precise-updates/icehouse',
71
'precise-icehouse': 'precise-updates/icehouse',
72
'precise-icehouse/updates': 'precise-updates/icehouse',
73
'precise-updates/icehouse': 'precise-updates/icehouse',
74
'icehouse/proposed': 'precise-proposed/icehouse',
75
'precise-icehouse/proposed': 'precise-proposed/icehouse',
76
'precise-proposed/icehouse': 'precise-proposed/icehouse',
78
'juno': 'trusty-updates/juno',
79
'trusty-juno': 'trusty-updates/juno',
80
'trusty-juno/updates': 'trusty-updates/juno',
81
'trusty-updates/juno': 'trusty-updates/juno',
82
'juno/proposed': 'trusty-proposed/juno',
83
'trusty-juno/proposed': 'trusty-proposed/juno',
84
'trusty-proposed/juno': 'trusty-proposed/juno',
86
'kilo': 'trusty-updates/kilo',
87
'trusty-kilo': 'trusty-updates/kilo',
88
'trusty-kilo/updates': 'trusty-updates/kilo',
89
'trusty-updates/kilo': 'trusty-updates/kilo',
90
'kilo/proposed': 'trusty-proposed/kilo',
91
'trusty-kilo/proposed': 'trusty-proposed/kilo',
92
'trusty-proposed/kilo': 'trusty-proposed/kilo',
95
# The order of this list is very important. Handlers should be listed in from
96
# least- to most-specific URL matching.
98
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
99
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
100
'charmhelpers.fetch.giturl.GitUrlFetchHandler',
103
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
104
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
105
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
108
class SourceConfigError(Exception):
112
class UnhandledSource(Exception):
116
class AptLockError(Exception):
120
class BaseFetchHandler(object):
122
"""Base class for FetchHandler implementations in fetch plugins"""
124
def can_handle(self, source):
125
"""Returns True if the source can be handled. Otherwise returns
126
a string explaining why it cannot"""
127
return "Wrong source type"
129
def install(self, source):
130
"""Try to download and unpack the source. Return the path to the
131
unpacked files or raise UnhandledSource."""
132
raise UnhandledSource("Wrong source type {}".format(source))
134
def parse_url(self, url):
137
def base_url(self, url):
138
"""Return url without querystring or fragment"""
139
parts = list(self.parse_url(url))
140
parts[4:] = ['' for i in parts[4:]]
141
return urlunparse(parts)
144
def filter_installed_packages(packages):
145
"""Returns a list of packages that require installation"""
148
for package in packages:
151
p.current_ver or _pkgs.append(package)
153
log('Package {} has no installation candidate.'.format(package),
155
_pkgs.append(package)
159
def apt_cache(in_memory=True):
160
"""Build and return an apt cache"""
161
from apt import apt_pkg
164
apt_pkg.config.set("Dir::Cache::pkgcache", "")
165
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
166
return apt_pkg.Cache()
169
def apt_install(packages, options=None, fatal=False):
170
"""Install one or more packages"""
172
options = ['--option=Dpkg::Options::=--force-confold']
174
cmd = ['apt-get', '--assume-yes']
176
cmd.append('install')
177
if isinstance(packages, six.string_types):
181
log("Installing {} with options: {}".format(packages,
183
_run_apt_command(cmd, fatal)
186
def apt_upgrade(options=None, fatal=False, dist=False):
187
"""Upgrade all packages"""
189
options = ['--option=Dpkg::Options::=--force-confold']
191
cmd = ['apt-get', '--assume-yes']
194
cmd.append('dist-upgrade')
196
cmd.append('upgrade')
197
log("Upgrading with options: {}".format(options))
198
_run_apt_command(cmd, fatal)
201
def apt_update(fatal=False):
202
"""Update local apt cache"""
203
cmd = ['apt-get', 'update']
204
_run_apt_command(cmd, fatal)
207
def apt_purge(packages, fatal=False):
208
"""Purge one or more packages"""
209
cmd = ['apt-get', '--assume-yes', 'purge']
210
if isinstance(packages, six.string_types):
214
log("Purging {}".format(packages))
215
_run_apt_command(cmd, fatal)
218
def apt_mark(packages, mark, fatal=False):
219
"""Flag one or more packages using apt-mark"""
220
cmd = ['apt-mark', mark]
221
if isinstance(packages, six.string_types):
225
log("Holding {}".format(packages))
228
subprocess.check_call(cmd, universal_newlines=True)
230
subprocess.call(cmd, universal_newlines=True)
233
def apt_hold(packages, fatal=False):
234
return apt_mark(packages, 'hold', fatal=fatal)
237
def apt_unhold(packages, fatal=False):
238
return apt_mark(packages, 'unhold', fatal=fatal)
241
def add_source(source, key=None):
242
"""Add a package source to this system.
244
@param source: a URL or sources.list entry, as supported by
245
add-apt-repository(1). Examples::
248
deb https://stub:key@private.example.com/ubuntu trusty main
251
'proposed:' may be used to enable the standard 'proposed'
252
pocket for the release.
253
'cloud:' may be used to activate official cloud archive pockets,
254
such as 'cloud:icehouse'
255
'distro' may be used as a noop
257
@param key: A key to be added to the system's APT keyring and used
258
to verify the signatures on packages. Ideally, this should be an
259
ASCII format GPG public key including the block headers. A GPG key
260
id may also be used, but be aware that only insecure protocols are
261
available to retrieve the actual public key from a public keyserver
262
placing your Juju environment at risk. ppa and cloud archive keys
263
are securely added automtically, so sould not be provided.
266
log('Source is not present. Skipping')
269
if (source.startswith('ppa:') or
270
source.startswith('http') or
271
source.startswith('deb ') or
272
source.startswith('cloud-archive:')):
273
subprocess.check_call(['add-apt-repository', '--yes', source])
274
elif source.startswith('cloud:'):
275
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
277
pocket = source.split(':')[-1]
278
if pocket not in CLOUD_ARCHIVE_POCKETS:
279
raise SourceConfigError(
280
'Unsupported cloud: source option %s' %
282
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
283
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
284
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
285
elif source == 'proposed':
286
release = lsb_release()['DISTRIB_CODENAME']
287
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
288
apt.write(PROPOSED_POCKET.format(release))
289
elif source == 'distro':
292
log("Unknown source: {!r}".format(source))
295
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
296
with NamedTemporaryFile('w+') as key_file:
300
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
302
# Note that hkp: is in no way a secure protocol. Using a
303
# GPG key id is pointless from a security POV unless you
304
# absolutely trust your network and DNS.
305
subprocess.check_call(['apt-key', 'adv', '--keyserver',
306
'hkp://keyserver.ubuntu.com:80', '--recv',
310
def configure_sources(update=False,
311
sources_var='install_sources',
312
keys_var='install_keys'):
314
Configure multiple sources from charm configuration.
316
The lists are encoded as yaml fragments in the configuration.
317
The frament needs to be included as a string. Sources and their
318
corresponding keys are of the types supported by add_source().
323
- "http://example.com/repo precise main"
328
Note that 'null' (a.k.a. None) should not be quoted.
330
sources = safe_load((config(sources_var) or '').strip()) or []
331
keys = safe_load((config(keys_var) or '').strip()) or None
333
if isinstance(sources, six.string_types):
337
for source in sources:
338
add_source(source, None)
340
if isinstance(keys, six.string_types):
343
if len(sources) != len(keys):
344
raise SourceConfigError(
345
'Install sources and keys lists are different lengths')
346
for source, key in zip(sources, keys):
347
add_source(source, key)
349
apt_update(fatal=True)
352
def install_remote(source, *args, **kwargs):
354
Install a file tree from a remote source
356
The specified source should be a url of the form:
357
scheme://[host]/path[#[option=value][&...]]
359
Schemes supported are based on this modules submodules.
360
Options supported are submodule-specific.
361
Additional arguments are passed through to the submodule.
365
dest = install_remote('http://example.com/archive.tgz',
369
This will download `archive.tgz`, validate it using SHA1 and, if
370
the file is ok, extract it and return the directory in which it
371
was extracted. If the checksum fails, it will raise
372
:class:`charmhelpers.core.host.ChecksumError`.
374
# We ONLY check for True here because can_handle may return a string
375
# explaining why it can't handle a given source.
376
handlers = [h for h in plugins() if h.can_handle(source) is True]
378
for handler in handlers:
380
installed_to = handler.install(source, *args, **kwargs)
381
except UnhandledSource as e:
382
log('Install source attempt unsuccessful: {}'.format(e),
385
raise UnhandledSource("No handler found for source {}".format(source))
389
def install_from_config(config_var_name):
390
charm_config = config()
391
source = charm_config[config_var_name]
392
return install_remote(source)
395
def plugins(fetch_handlers=None):
396
if not fetch_handlers:
397
fetch_handlers = FETCH_HANDLERS
399
for handler_name in fetch_handlers:
400
package, classname = handler_name.rsplit('.', 1)
402
handler_class = getattr(
403
importlib.import_module(package),
405
plugin_list.append(handler_class())
406
except (ImportError, AttributeError):
407
# Skip missing plugins so that they can be ommitted from
408
# installation if desired
409
log("FetchHandler {} not found, skipping plugin".format(
414
def _run_apt_command(cmd, fatal=False):
416
Run an APT command, checking output and retrying if the fatal flag is set
419
:param: cmd: str: The apt command to run.
420
:param: fatal: bool: Whether the command's output should be checked and
423
env = os.environ.copy()
425
if 'DEBIAN_FRONTEND' not in env:
426
env['DEBIAN_FRONTEND'] = 'noninteractive'
432
# If the command is considered "fatal", we need to retry if the apt
433
# lock was not acquired.
435
while result is None or result == APT_NO_LOCK:
437
result = subprocess.check_call(cmd, env=env)
438
except subprocess.CalledProcessError as e:
439
retry_count = retry_count + 1
440
if retry_count > APT_NO_LOCK_RETRY_COUNT:
442
result = e.returncode
443
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
444
"".format(APT_NO_LOCK_RETRY_DELAY))
445
time.sleep(APT_NO_LOCK_RETRY_DELAY)
448
subprocess.call(cmd, env=env)
added
removed
hooks/charmhelpers/fetch/__init__.py
