2
from tempfile import NamedTemporaryFile
4
from yaml import safe_load
5
from charmhelpers.core.host import (
13
from charmhelpers.core.hookenv import (
20
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
21
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
23
PROPOSED_POCKET = """# Proposed
24
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
26
CLOUD_ARCHIVE_POCKETS = {
28
'folsom': 'precise-updates/folsom',
29
'precise-folsom': 'precise-updates/folsom',
30
'precise-folsom/updates': 'precise-updates/folsom',
31
'precise-updates/folsom': 'precise-updates/folsom',
32
'folsom/proposed': 'precise-proposed/folsom',
33
'precise-folsom/proposed': 'precise-proposed/folsom',
34
'precise-proposed/folsom': 'precise-proposed/folsom',
36
'grizzly': 'precise-updates/grizzly',
37
'precise-grizzly': 'precise-updates/grizzly',
38
'precise-grizzly/updates': 'precise-updates/grizzly',
39
'precise-updates/grizzly': 'precise-updates/grizzly',
40
'grizzly/proposed': 'precise-proposed/grizzly',
41
'precise-grizzly/proposed': 'precise-proposed/grizzly',
42
'precise-proposed/grizzly': 'precise-proposed/grizzly',
44
'havana': 'precise-updates/havana',
45
'precise-havana': 'precise-updates/havana',
46
'precise-havana/updates': 'precise-updates/havana',
47
'precise-updates/havana': 'precise-updates/havana',
48
'havana/proposed': 'precise-proposed/havana',
49
'precise-havana/proposed': 'precise-proposed/havana',
50
'precise-proposed/havana': 'precise-proposed/havana',
52
'icehouse': 'precise-updates/icehouse',
53
'precise-icehouse': 'precise-updates/icehouse',
54
'precise-icehouse/updates': 'precise-updates/icehouse',
55
'precise-updates/icehouse': 'precise-updates/icehouse',
56
'icehouse/proposed': 'precise-proposed/icehouse',
57
'precise-icehouse/proposed': 'precise-proposed/icehouse',
58
'precise-proposed/icehouse': 'precise-proposed/icehouse',
60
'juno': 'trusty-updates/juno',
61
'trusty-juno': 'trusty-updates/juno',
62
'trusty-juno/updates': 'trusty-updates/juno',
63
'trusty-updates/juno': 'trusty-updates/juno',
64
'juno/proposed': 'trusty-proposed/juno',
65
'juno/proposed': 'trusty-proposed/juno',
66
'trusty-juno/proposed': 'trusty-proposed/juno',
67
'trusty-proposed/juno': 'trusty-proposed/juno',
70
# The order of this list is very important. Handlers should be listed in from
71
# least- to most-specific URL matching.
73
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
74
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
77
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
78
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
79
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
82
class SourceConfigError(Exception):
86
class UnhandledSource(Exception):
90
class AptLockError(Exception):
94
class BaseFetchHandler(object):
96
"""Base class for FetchHandler implementations in fetch plugins"""
98
def can_handle(self, source):
99
"""Returns True if the source can be handled. Otherwise returns
100
a string explaining why it cannot"""
101
return "Wrong source type"
103
def install(self, source):
104
"""Try to download and unpack the source. Return the path to the
105
unpacked files or raise UnhandledSource."""
106
raise UnhandledSource("Wrong source type {}".format(source))
108
def parse_url(self, url):
111
def base_url(self, url):
112
"""Return url without querystring or fragment"""
113
parts = list(self.parse_url(url))
114
parts[4:] = ['' for i in parts[4:]]
115
return urlunparse(parts)
118
def filter_installed_packages(packages):
119
"""Returns a list of packages that require installation"""
122
for package in packages:
125
p.current_ver or _pkgs.append(package)
127
log('Package {} has no installation candidate.'.format(package),
129
_pkgs.append(package)
133
def apt_cache(in_memory=True):
134
"""Build and return an apt cache"""
138
apt_pkg.config.set("Dir::Cache::pkgcache", "")
139
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
140
return apt_pkg.Cache()
143
def apt_install(packages, options=None, fatal=False):
144
"""Install one or more packages"""
146
options = ['--option=Dpkg::Options::=--force-confold']
148
cmd = ['apt-get', '--assume-yes']
150
cmd.append('install')
151
if isinstance(packages, basestring):
155
log("Installing {} with options: {}".format(packages,
157
_run_apt_command(cmd, fatal)
160
def apt_upgrade(options=None, fatal=False, dist=False):
161
"""Upgrade all packages"""
163
options = ['--option=Dpkg::Options::=--force-confold']
165
cmd = ['apt-get', '--assume-yes']
168
cmd.append('dist-upgrade')
170
cmd.append('upgrade')
171
log("Upgrading with options: {}".format(options))
172
_run_apt_command(cmd, fatal)
175
def apt_update(fatal=False):
176
"""Update local apt cache"""
177
cmd = ['apt-get', 'update']
178
_run_apt_command(cmd, fatal)
181
def apt_purge(packages, fatal=False):
182
"""Purge one or more packages"""
183
cmd = ['apt-get', '--assume-yes', 'purge']
184
if isinstance(packages, basestring):
188
log("Purging {}".format(packages))
189
_run_apt_command(cmd, fatal)
192
def apt_hold(packages, fatal=False):
193
"""Hold one or more packages"""
194
cmd = ['apt-mark', 'hold']
195
if isinstance(packages, basestring):
199
log("Holding {}".format(packages))
202
subprocess.check_call(cmd)
207
def add_source(source, key=None):
208
"""Add a package source to this system.
210
@param source: a URL or sources.list entry, as supported by
211
add-apt-repository(1). Examples:
213
deb https://stub:key@private.example.com/ubuntu trusty main
216
'proposed:' may be used to enable the standard 'proposed'
217
pocket for the release.
218
'cloud:' may be used to activate official cloud archive pockets,
219
such as 'cloud:icehouse'
221
@param key: A key to be added to the system's APT keyring and used
222
to verify the signatures on packages. Ideally, this should be an
223
ASCII format GPG public key including the block headers. A GPG key
224
id may also be used, but be aware that only insecure protocols are
225
available to retrieve the actual public key from a public keyserver
226
placing your Juju environment at risk. ppa and cloud archive keys
227
are securely added automtically, so sould not be provided.
230
log('Source is not present. Skipping')
233
if (source.startswith('ppa:') or
234
source.startswith('http') or
235
source.startswith('deb ') or
236
source.startswith('cloud-archive:')):
237
subprocess.check_call(['add-apt-repository', '--yes', source])
238
elif source.startswith('cloud:'):
239
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
241
pocket = source.split(':')[-1]
242
if pocket not in CLOUD_ARCHIVE_POCKETS:
243
raise SourceConfigError(
244
'Unsupported cloud: source option %s' %
246
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
247
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
248
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
249
elif source == 'proposed':
250
release = lsb_release()['DISTRIB_CODENAME']
251
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
252
apt.write(PROPOSED_POCKET.format(release))
254
raise SourceConfigError("Unknown source: {!r}".format(source))
257
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
258
with NamedTemporaryFile() as key_file:
262
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
264
# Note that hkp: is in no way a secure protocol. Using a
265
# GPG key id is pointless from a security POV unless you
266
# absolutely trust your network and DNS.
267
subprocess.check_call(['apt-key', 'adv', '--keyserver',
268
'hkp://keyserver.ubuntu.com:80', '--recv',
272
def configure_sources(update=False,
273
sources_var='install_sources',
274
keys_var='install_keys'):
276
Configure multiple sources from charm configuration.
278
The lists are encoded as yaml fragments in the configuration.
279
The frament needs to be included as a string. Sources and their
280
corresponding keys are of the types supported by add_source().
285
- "http://example.com/repo precise main"
290
Note that 'null' (a.k.a. None) should not be quoted.
292
sources = safe_load((config(sources_var) or '').strip()) or []
293
keys = safe_load((config(keys_var) or '').strip()) or None
295
if isinstance(sources, basestring):
299
for source in sources:
300
add_source(source, None)
302
if isinstance(keys, basestring):
305
if len(sources) != len(keys):
306
raise SourceConfigError(
307
'Install sources and keys lists are different lengths')
308
for source, key in zip(sources, keys):
309
add_source(source, key)
311
apt_update(fatal=True)
314
def install_remote(source):
316
Install a file tree from a remote source
318
The specified source should be a url of the form:
319
scheme://[host]/path[#[option=value][&...]]
321
Schemes supported are based on this modules submodules
322
Options supported are submodule-specific"""
323
# We ONLY check for True here because can_handle may return a string
324
# explaining why it can't handle a given source.
325
handlers = [h for h in plugins() if h.can_handle(source) is True]
327
for handler in handlers:
329
installed_to = handler.install(source)
330
except UnhandledSource:
333
raise UnhandledSource("No handler found for source {}".format(source))
337
def install_from_config(config_var_name):
338
charm_config = config()
339
source = charm_config[config_var_name]
340
return install_remote(source)
343
def plugins(fetch_handlers=None):
344
if not fetch_handlers:
345
fetch_handlers = FETCH_HANDLERS
347
for handler_name in fetch_handlers:
348
package, classname = handler_name.rsplit('.', 1)
350
handler_class = getattr(
351
importlib.import_module(package),
353
plugin_list.append(handler_class())
354
except (ImportError, AttributeError):
355
# Skip missing plugins so that they can be ommitted from
356
# installation if desired
357
log("FetchHandler {} not found, skipping plugin".format(
362
def _run_apt_command(cmd, fatal=False):
364
Run an APT command, checking output and retrying if the fatal flag is set
367
:param: cmd: str: The apt command to run.
368
:param: fatal: bool: Whether the command's output should be checked and
371
env = os.environ.copy()
373
if 'DEBIAN_FRONTEND' not in env:
374
env['DEBIAN_FRONTEND'] = 'noninteractive'
380
# If the command is considered "fatal", we need to retry if the apt
381
# lock was not acquired.
383
while result is None or result == APT_NO_LOCK:
385
result = subprocess.check_call(cmd, env=env)
386
except subprocess.CalledProcessError, e:
387
retry_count = retry_count + 1
388
if retry_count > APT_NO_LOCK_RETRY_COUNT:
390
result = e.returncode
391
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
392
"".format(APT_NO_LOCK_RETRY_DELAY))
393
time.sleep(APT_NO_LOCK_RETRY_DELAY)
396
subprocess.call(cmd, env=env)
added
removed
hooks/charmhelpers/fetch/__init__.py
