« back to all changes in this revision
Viewing changes to hooks/charmhelpers/fetch/__init__.py
- browse files at revision 11
- compare with another revision
- download diff
- download tarball
- view history from revision 11
- Committer: Tim Van Steenburgh
- Date: 2015-05-01 10:12:08 UTC
- mfrom: (10.1.4 ubuntu)
- Revision ID: tim.van.steenburgh@canonical.com-20150501101208-7jvqc9dtmlfz7kvk
[1chb1n] Remove hooks and lxc stuff; update tests
Revert charm to have no hooks and no config options; add functional tests for all currently-supported Ubuntu releases; add bundletester usage examples.
Charm is now compatible with both 'juju test' and bundletester.
Revert charm to have no hooks and no config options; add functional tests for all currently-supported Ubuntu releases; add bundletester usage examples.
Charm is now compatible with both 'juju test' and bundletester.
- files added:
- Makefile
- files
- tests/charmhelpers
- tests/charmhelpers/contrib
- tests/charmhelpers/contrib/amulet
- files removed:
- charm-helpers-hooks.yaml
- hooks
- hooks/charmhelpers
- hooks/charmhelpers/core
- hooks/charmhelpers/core/services
- hooks/charmhelpers/fetch
- templates
- files modified:
- README.md
added
removed
hooks/charmhelpers/fetch/__init__.py
1
# Copyright 2014-2015 Canonical Limited.
2
#
3
# This file is part of charm-helpers.
4
#
5
# charm-helpers is free software: you can redistribute it and/or modify
6
# it under the terms of the GNU Lesser General Public License version 3 as
7
# published by the Free Software Foundation.
8
#
9
# charm-helpers is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU Lesser General Public License for more details.
13
#
14
# You should have received a copy of the GNU Lesser General Public License
15
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17
import importlib
18
from tempfile import NamedTemporaryFile
19
import time
20
from yaml import safe_load
21
from charmhelpers.core.host import (
22
lsb_release
23
)
24
import subprocess
25
from charmhelpers.core.hookenv import (
26
config,
27
log,
28
)
29
import os
30
31
import six
32
if six.PY3:
33
from urllib.parse import urlparse, urlunparse
34
else:
35
from urlparse import urlparse, urlunparse
36
37
38
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
39
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
40
"""
41
PROPOSED_POCKET = """# Proposed
42
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
43
"""
44
CLOUD_ARCHIVE_POCKETS = {
45
# Folsom
46
'folsom': 'precise-updates/folsom',
47
'precise-folsom': 'precise-updates/folsom',
48
'precise-folsom/updates': 'precise-updates/folsom',
49
'precise-updates/folsom': 'precise-updates/folsom',
50
'folsom/proposed': 'precise-proposed/folsom',
51
'precise-folsom/proposed': 'precise-proposed/folsom',
52
'precise-proposed/folsom': 'precise-proposed/folsom',
53
# Grizzly
54
'grizzly': 'precise-updates/grizzly',
55
'precise-grizzly': 'precise-updates/grizzly',
56
'precise-grizzly/updates': 'precise-updates/grizzly',
57
'precise-updates/grizzly': 'precise-updates/grizzly',
58
'grizzly/proposed': 'precise-proposed/grizzly',
59
'precise-grizzly/proposed': 'precise-proposed/grizzly',
60
'precise-proposed/grizzly': 'precise-proposed/grizzly',
61
# Havana
62
'havana': 'precise-updates/havana',
63
'precise-havana': 'precise-updates/havana',
64
'precise-havana/updates': 'precise-updates/havana',
65
'precise-updates/havana': 'precise-updates/havana',
66
'havana/proposed': 'precise-proposed/havana',
67
'precise-havana/proposed': 'precise-proposed/havana',
68
'precise-proposed/havana': 'precise-proposed/havana',
69
# Icehouse
70
'icehouse': 'precise-updates/icehouse',
71
'precise-icehouse': 'precise-updates/icehouse',
72
'precise-icehouse/updates': 'precise-updates/icehouse',
73
'precise-updates/icehouse': 'precise-updates/icehouse',
74
'icehouse/proposed': 'precise-proposed/icehouse',
75
'precise-icehouse/proposed': 'precise-proposed/icehouse',
76
'precise-proposed/icehouse': 'precise-proposed/icehouse',
77
# Juno
78
'juno': 'trusty-updates/juno',
79
'trusty-juno': 'trusty-updates/juno',
80
'trusty-juno/updates': 'trusty-updates/juno',
81
'trusty-updates/juno': 'trusty-updates/juno',
82
'juno/proposed': 'trusty-proposed/juno',
83
'trusty-juno/proposed': 'trusty-proposed/juno',
84
'trusty-proposed/juno': 'trusty-proposed/juno',
85
# Kilo
86
'kilo': 'trusty-updates/kilo',
87
'trusty-kilo': 'trusty-updates/kilo',
88
'trusty-kilo/updates': 'trusty-updates/kilo',
89
'trusty-updates/kilo': 'trusty-updates/kilo',
90
'kilo/proposed': 'trusty-proposed/kilo',
91
'trusty-kilo/proposed': 'trusty-proposed/kilo',
92
'trusty-proposed/kilo': 'trusty-proposed/kilo',
93
}
94
95
# The order of this list is very important. Handlers should be listed in from
96
# least- to most-specific URL matching.
97
FETCH_HANDLERS = (
98
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
99
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
100
'charmhelpers.fetch.giturl.GitUrlFetchHandler',
101
)
102
103
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
104
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
105
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
106
107
108
class SourceConfigError(Exception):
109
pass
110
111
112
class UnhandledSource(Exception):
113
pass
114
115
116
class AptLockError(Exception):
117
pass
118
119
120
class BaseFetchHandler(object):
121
122
"""Base class for FetchHandler implementations in fetch plugins"""
123
124
def can_handle(self, source):
125
"""Returns True if the source can be handled. Otherwise returns
126
a string explaining why it cannot"""
127
return "Wrong source type"
128
129
def install(self, source):
130
"""Try to download and unpack the source. Return the path to the
131
unpacked files or raise UnhandledSource."""
132
raise UnhandledSource("Wrong source type {}".format(source))
133
134
def parse_url(self, url):
135
return urlparse(url)
136
137
def base_url(self, url):
138
"""Return url without querystring or fragment"""
139
parts = list(self.parse_url(url))
140
parts[4:] = ['' for i in parts[4:]]
141
return urlunparse(parts)
142
143
144
def filter_installed_packages(packages):
145
"""Returns a list of packages that require installation"""
146
cache = apt_cache()
147
_pkgs = []
148
for package in packages:
149
try:
150
p = cache[package]
151
p.current_ver or _pkgs.append(package)
152
except KeyError:
153
log('Package {} has no installation candidate.'.format(package),
154
level='WARNING')
155
_pkgs.append(package)
156
return _pkgs
157
158
159
def apt_cache(in_memory=True):
160
"""Build and return an apt cache"""
161
import apt_pkg
162
apt_pkg.init()
163
if in_memory:
164
apt_pkg.config.set("Dir::Cache::pkgcache", "")
165
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
166
return apt_pkg.Cache()
167
168
169
def apt_install(packages, options=None, fatal=False):
170
"""Install one or more packages"""
171
if options is None:
172
options = ['--option=Dpkg::Options::=--force-confold']
173
174
cmd = ['apt-get', '--assume-yes']
175
cmd.extend(options)
176
cmd.append('install')
177
if isinstance(packages, six.string_types):
178
cmd.append(packages)
179
else:
180
cmd.extend(packages)
181
log("Installing {} with options: {}".format(packages,
182
options))
183
_run_apt_command(cmd, fatal)
184
185
186
def apt_upgrade(options=None, fatal=False, dist=False):
187
"""Upgrade all packages"""
188
if options is None:
189
options = ['--option=Dpkg::Options::=--force-confold']
190
191
cmd = ['apt-get', '--assume-yes']
192
cmd.extend(options)
193
if dist:
194
cmd.append('dist-upgrade')
195
else:
196
cmd.append('upgrade')
197
log("Upgrading with options: {}".format(options))
198
_run_apt_command(cmd, fatal)
199
200
201
def apt_update(fatal=False):
202
"""Update local apt cache"""
203
cmd = ['apt-get', 'update']
204
_run_apt_command(cmd, fatal)
205
206
207
def apt_purge(packages, fatal=False):
208
"""Purge one or more packages"""
209
cmd = ['apt-get', '--assume-yes', 'purge']
210
if isinstance(packages, six.string_types):
211
cmd.append(packages)
212
else:
213
cmd.extend(packages)
214
log("Purging {}".format(packages))
215
_run_apt_command(cmd, fatal)
216
217
218
def apt_hold(packages, fatal=False):
219
"""Hold one or more packages"""
220
cmd = ['apt-mark', 'hold']
221
if isinstance(packages, six.string_types):
222
cmd.append(packages)
223
else:
224
cmd.extend(packages)
225
log("Holding {}".format(packages))
226
227
if fatal:
228
subprocess.check_call(cmd)
229
else:
230
subprocess.call(cmd)
231
232
233
def add_source(source, key=None):
234
"""Add a package source to this system.
235
236
@param source: a URL or sources.list entry, as supported by
237
add-apt-repository(1). Examples::
238
239
ppa:charmers/example
240
deb https://stub:key@private.example.com/ubuntu trusty main
241
242
In addition:
243
'proposed:' may be used to enable the standard 'proposed'
244
pocket for the release.
245
'cloud:' may be used to activate official cloud archive pockets,
246
such as 'cloud:icehouse'
247
'distro' may be used as a noop
248
249
@param key: A key to be added to the system's APT keyring and used
250
to verify the signatures on packages. Ideally, this should be an
251
ASCII format GPG public key including the block headers. A GPG key
252
id may also be used, but be aware that only insecure protocols are
253
available to retrieve the actual public key from a public keyserver
254
placing your Juju environment at risk. ppa and cloud archive keys
255
are securely added automtically, so sould not be provided.
256
"""
257
if source is None:
258
log('Source is not present. Skipping')
259
return
260
261
if (source.startswith('ppa:') or
262
source.startswith('http') or
263
source.startswith('deb ') or
264
source.startswith('cloud-archive:')):
265
subprocess.check_call(['add-apt-repository', '--yes', source])
266
elif source.startswith('cloud:'):
267
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
268
fatal=True)
269
pocket = source.split(':')[-1]
270
if pocket not in CLOUD_ARCHIVE_POCKETS:
271
raise SourceConfigError(
272
'Unsupported cloud: source option %s' %
273
pocket)
274
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
275
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
276
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
277
elif source == 'proposed':
278
release = lsb_release()['DISTRIB_CODENAME']
279
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
280
apt.write(PROPOSED_POCKET.format(release))
281
elif source == 'distro':
282
pass
283
else:
284
log("Unknown source: {!r}".format(source))
285
286
if key:
287
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
288
with NamedTemporaryFile('w+') as key_file:
289
key_file.write(key)
290
key_file.flush()
291
key_file.seek(0)
292
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
293
else:
294
# Note that hkp: is in no way a secure protocol. Using a
295
# GPG key id is pointless from a security POV unless you
296
# absolutely trust your network and DNS.
297
subprocess.check_call(['apt-key', 'adv', '--keyserver',
298
'hkp://keyserver.ubuntu.com:80', '--recv',
299
key])
300
301
302
def configure_sources(update=False,
303
sources_var='install_sources',
304
keys_var='install_keys'):
305
"""
306
Configure multiple sources from charm configuration.
307
308
The lists are encoded as yaml fragments in the configuration.
309
The frament needs to be included as a string. Sources and their
310
corresponding keys are of the types supported by add_source().
311
312
Example config:
313
install_sources: |
314
- "ppa:foo"
315
- "http://example.com/repo precise main"
316
install_keys: |
317
- null
318
- "a1b2c3d4"
319
320
Note that 'null' (a.k.a. None) should not be quoted.
321
"""
322
sources = safe_load((config(sources_var) or '').strip()) or []
323
keys = safe_load((config(keys_var) or '').strip()) or None
324
325
if isinstance(sources, six.string_types):
326
sources = [sources]
327
328
if keys is None:
329
for source in sources:
330
add_source(source, None)
331
else:
332
if isinstance(keys, six.string_types):
333
keys = [keys]
334
335
if len(sources) != len(keys):
336
raise SourceConfigError(
337
'Install sources and keys lists are different lengths')
338
for source, key in zip(sources, keys):
339
add_source(source, key)
340
if update:
341
apt_update(fatal=True)
342
343
344
def install_remote(source, *args, **kwargs):
345
"""
346
Install a file tree from a remote source
347
348
The specified source should be a url of the form:
349
scheme://[host]/path[#[option=value][&...]]
350
351
Schemes supported are based on this modules submodules.
352
Options supported are submodule-specific.
353
Additional arguments are passed through to the submodule.
354
355
For example::
356
357
dest = install_remote('http://example.com/archive.tgz',
358
checksum='deadbeef',
359
hash_type='sha1')
360
361
This will download `archive.tgz`, validate it using SHA1 and, if
362
the file is ok, extract it and return the directory in which it
363
was extracted. If the checksum fails, it will raise
364
:class:`charmhelpers.core.host.ChecksumError`.
365
"""
366
# We ONLY check for True here because can_handle may return a string
367
# explaining why it can't handle a given source.
368
handlers = [h for h in plugins() if h.can_handle(source) is True]
369
installed_to = None
370
for handler in handlers:
371
try:
372
installed_to = handler.install(source, *args, **kwargs)
373
except UnhandledSource:
374
pass
375
if not installed_to:
376
raise UnhandledSource("No handler found for source {}".format(source))
377
return installed_to
378
379
380
def install_from_config(config_var_name):
381
charm_config = config()
382
source = charm_config[config_var_name]
383
return install_remote(source)
384
385
386
def plugins(fetch_handlers=None):
387
if not fetch_handlers:
388
fetch_handlers = FETCH_HANDLERS
389
plugin_list = []
390
for handler_name in fetch_handlers:
391
package, classname = handler_name.rsplit('.', 1)
392
try:
393
handler_class = getattr(
394
importlib.import_module(package),
395
classname)
396
plugin_list.append(handler_class())
397
except (ImportError, AttributeError):
398
# Skip missing plugins so that they can be ommitted from
399
# installation if desired
400
log("FetchHandler {} not found, skipping plugin".format(
401
handler_name))
402
return plugin_list
403
404
405
def _run_apt_command(cmd, fatal=False):
406
"""
407
Run an APT command, checking output and retrying if the fatal flag is set
408
to True.
409
410
:param: cmd: str: The apt command to run.
411
:param: fatal: bool: Whether the command's output should be checked and
412
retried.
413
"""
414
env = os.environ.copy()
415
416
if 'DEBIAN_FRONTEND' not in env:
417
env['DEBIAN_FRONTEND'] = 'noninteractive'
418
419
if fatal:
420
retry_count = 0
421
result = None
422
423
# If the command is considered "fatal", we need to retry if the apt
424
# lock was not acquired.
425
426
while result is None or result == APT_NO_LOCK:
427
try:
428
result = subprocess.check_call(cmd, env=env)
429
except subprocess.CalledProcessError as e:
430
retry_count = retry_count + 1
431
if retry_count > APT_NO_LOCK_RETRY_COUNT:
432
raise
433
result = e.returncode
434
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
435
"".format(APT_NO_LOCK_RETRY_DELAY))
436
time.sleep(APT_NO_LOCK_RETRY_DELAY)
437
438
else:
439
subprocess.call(cmd, env=env)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1