« back to all changes in this revision
Viewing changes to hooks/charmhelpers/fetch/__init__.py
- browse files at revision 4
- compare with another revision
- download diff
- download tarball
- view history from revision 4
- Committer: Jill Rouleau
- Date: 2015-04-14 18:50:35 UTC
- Revision ID: jill.rouleau@canonical.com-20150414185035-8hloa3d8aw33lkre
sync-charm-helpers, add hooks
- files added:
- hooks/charmhelpers/core/decorators.py
- files modified:
- charm-helpers.yaml
added
removed
hooks/charmhelpers/fetch/__init__.py
1
# Copyright 2014-2015 Canonical Limited.
2
#
3
# This file is part of charm-helpers.
4
#
5
# charm-helpers is free software: you can redistribute it and/or modify
6
# it under the terms of the GNU Lesser General Public License version 3 as
7
# published by the Free Software Foundation.
8
#
9
# charm-helpers is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU Lesser General Public License for more details.
13
#
14
# You should have received a copy of the GNU Lesser General Public License
15
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1
17
import importlib
18
from tempfile import NamedTemporaryFile
2
19
import time
3
20
from yaml import safe_load
4
21
from charmhelpers.core.host import (
5
22
lsb_release
6
23
)
7
from urlparse import (
8
urlparse,
9
urlunparse,
10
)
11
24
import subprocess
12
25
from charmhelpers.core.hookenv import (
13
26
config,
14
27
log,
15
28
)
16
import apt_pkg
17
29
import os
18
30
31
import six
32
if six.PY3:
33
from urllib.parse import urlparse, urlunparse
34
else:
35
from urlparse import urlparse, urlunparse
36
19
37
20
38
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
21
39
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
56
74
'icehouse/proposed': 'precise-proposed/icehouse',
57
75
'precise-icehouse/proposed': 'precise-proposed/icehouse',
58
76
'precise-proposed/icehouse': 'precise-proposed/icehouse',
77
# Juno
78
'juno': 'trusty-updates/juno',
79
'trusty-juno': 'trusty-updates/juno',
80
'trusty-juno/updates': 'trusty-updates/juno',
81
'trusty-updates/juno': 'trusty-updates/juno',
82
'juno/proposed': 'trusty-proposed/juno',
83
'trusty-juno/proposed': 'trusty-proposed/juno',
84
'trusty-proposed/juno': 'trusty-proposed/juno',
85
# Kilo
86
'kilo': 'trusty-updates/kilo',
87
'trusty-kilo': 'trusty-updates/kilo',
88
'trusty-kilo/updates': 'trusty-updates/kilo',
89
'trusty-updates/kilo': 'trusty-updates/kilo',
90
'kilo/proposed': 'trusty-proposed/kilo',
91
'trusty-kilo/proposed': 'trusty-proposed/kilo',
92
'trusty-proposed/kilo': 'trusty-proposed/kilo',
59
93
}
60
94
61
95
# The order of this list is very important. Handlers should be listed in from
63
97
FETCH_HANDLERS = (
64
98
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
65
99
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
100
'charmhelpers.fetch.giturl.GitUrlFetchHandler',
66
101
)
67
102
68
103
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
108
143
109
144
def filter_installed_packages(packages):
110
145
"""Returns a list of packages that require installation"""
111
apt_pkg.init()
112
113
# Tell apt to build an in-memory cache to prevent race conditions (if
114
# another process is already building the cache).
115
apt_pkg.config.set("Dir::Cache::pkgcache", "")
116
117
cache = apt_pkg.Cache()
146
cache = apt_cache()
118
147
_pkgs = []
119
148
for package in packages:
120
149
try:
127
156
return _pkgs
128
157
129
158
159
def apt_cache(in_memory=True):
160
"""Build and return an apt cache"""
161
import apt_pkg
162
apt_pkg.init()
163
if in_memory:
164
apt_pkg.config.set("Dir::Cache::pkgcache", "")
165
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
166
return apt_pkg.Cache()
167
168
130
169
def apt_install(packages, options=None, fatal=False):
131
170
"""Install one or more packages"""
132
171
if options is None:
135
174
cmd = ['apt-get', '--assume-yes']
136
175
cmd.extend(options)
137
176
cmd.append('install')
138
if isinstance(packages, basestring):
177
if isinstance(packages, six.string_types):
139
178
cmd.append(packages)
140
179
else:
141
180
cmd.extend(packages)
168
207
def apt_purge(packages, fatal=False):
169
208
"""Purge one or more packages"""
170
209
cmd = ['apt-get', '--assume-yes', 'purge']
171
if isinstance(packages, basestring):
210
if isinstance(packages, six.string_types):
172
211
cmd.append(packages)
173
212
else:
174
213
cmd.extend(packages)
179
218
def apt_hold(packages, fatal=False):
180
219
"""Hold one or more packages"""
181
220
cmd = ['apt-mark', 'hold']
182
if isinstance(packages, basestring):
221
if isinstance(packages, six.string_types):
183
222
cmd.append(packages)
184
223
else:
185
224
cmd.extend(packages)
192
231
193
232
194
233
def add_source(source, key=None):
234
"""Add a package source to this system.
235
236
@param source: a URL or sources.list entry, as supported by
237
add-apt-repository(1). Examples::
238
239
ppa:charmers/example
240
deb https://stub:key@private.example.com/ubuntu trusty main
241
242
In addition:
243
'proposed:' may be used to enable the standard 'proposed'
244
pocket for the release.
245
'cloud:' may be used to activate official cloud archive pockets,
246
such as 'cloud:icehouse'
247
'distro' may be used as a noop
248
249
@param key: A key to be added to the system's APT keyring and used
250
to verify the signatures on packages. Ideally, this should be an
251
ASCII format GPG public key including the block headers. A GPG key
252
id may also be used, but be aware that only insecure protocols are
253
available to retrieve the actual public key from a public keyserver
254
placing your Juju environment at risk. ppa and cloud archive keys
255
are securely added automtically, so sould not be provided.
256
"""
195
257
if source is None:
196
258
log('Source is not present. Skipping')
197
259
return
216
278
release = lsb_release()['DISTRIB_CODENAME']
217
279
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
218
280
apt.write(PROPOSED_POCKET.format(release))
281
elif source == 'distro':
282
pass
283
else:
284
log("Unknown source: {!r}".format(source))
285
219
286
if key:
220
subprocess.check_call(['apt-key', 'adv', '--keyserver',
221
'hkp://keyserver.ubuntu.com:80', '--recv',
222
key])
287
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
288
with NamedTemporaryFile('w+') as key_file:
289
key_file.write(key)
290
key_file.flush()
291
key_file.seek(0)
292
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
293
else:
294
# Note that hkp: is in no way a secure protocol. Using a
295
# GPG key id is pointless from a security POV unless you
296
# absolutely trust your network and DNS.
297
subprocess.check_call(['apt-key', 'adv', '--keyserver',
298
'hkp://keyserver.ubuntu.com:80', '--recv',
299
key])
223
300
224
301
225
302
def configure_sources(update=False,
226
303
sources_var='install_sources',
227
304
keys_var='install_keys'):
228
305
"""
229
Configure multiple sources from charm configuration
306
Configure multiple sources from charm configuration.
307
308
The lists are encoded as yaml fragments in the configuration.
309
The frament needs to be included as a string. Sources and their
310
corresponding keys are of the types supported by add_source().
230
311
231
312
Example config:
232
install_sources:
313
install_sources: |
233
314
- "ppa:foo"
234
315
- "http://example.com/repo precise main"
235
install_keys:
316
install_keys: |
236
317
- null
237
318
- "a1b2c3d4"
238
319
239
320
Note that 'null' (a.k.a. None) should not be quoted.
240
321
"""
241
sources = safe_load(config(sources_var))
242
keys = config(keys_var)
243
if keys is not None:
244
keys = safe_load(keys)
245
if isinstance(sources, basestring) and (
246
keys is None or isinstance(keys, basestring)):
247
add_source(sources, keys)
322
sources = safe_load((config(sources_var) or '').strip()) or []
323
keys = safe_load((config(keys_var) or '').strip()) or None
324
325
if isinstance(sources, six.string_types):
326
sources = [sources]
327
328
if keys is None:
329
for source in sources:
330
add_source(source, None)
248
331
else:
249
if not len(sources) == len(keys):
250
msg = 'Install sources and keys lists are different lengths'
251
raise SourceConfigError(msg)
252
for src_num in range(len(sources)):
253
add_source(sources[src_num], keys[src_num])
332
if isinstance(keys, six.string_types):
333
keys = [keys]
334
335
if len(sources) != len(keys):
336
raise SourceConfigError(
337
'Install sources and keys lists are different lengths')
338
for source, key in zip(sources, keys):
339
add_source(source, key)
254
340
if update:
255
341
apt_update(fatal=True)
256
342
257
343
258
def install_remote(source):
344
def install_remote(source, *args, **kwargs):
259
345
"""
260
346
Install a file tree from a remote source
261
347
262
348
The specified source should be a url of the form:
263
349
scheme://[host]/path[#[option=value][&...]]
264
350
265
Schemes supported are based on this modules submodules
266
Options supported are submodule-specific"""
351
Schemes supported are based on this modules submodules.
352
Options supported are submodule-specific.
353
Additional arguments are passed through to the submodule.
354
355
For example::
356
357
dest = install_remote('http://example.com/archive.tgz',
358
checksum='deadbeef',
359
hash_type='sha1')
360
361
This will download `archive.tgz`, validate it using SHA1 and, if
362
the file is ok, extract it and return the directory in which it
363
was extracted. If the checksum fails, it will raise
364
:class:`charmhelpers.core.host.ChecksumError`.
365
"""
267
366
# We ONLY check for True here because can_handle may return a string
268
367
# explaining why it can't handle a given source.
269
368
handlers = [h for h in plugins() if h.can_handle(source) is True]
270
369
installed_to = None
271
370
for handler in handlers:
272
371
try:
273
installed_to = handler.install(source)
372
installed_to = handler.install(source, *args, **kwargs)
274
373
except UnhandledSource:
275
374
pass
276
375
if not installed_to:
327
426
while result is None or result == APT_NO_LOCK:
328
427
try:
329
428
result = subprocess.check_call(cmd, env=env)
330
except subprocess.CalledProcessError, e:
429
except subprocess.CalledProcessError as e:
331
430
retry_count = retry_count + 1
332
431
if retry_count > APT_NO_LOCK_RETRY_COUNT:
333
432
raise
Loggerhead is a web-based interface for Breezy
Version: 2.0.1