~ajkavanagh/charms/trusty/memcached/add-spaces-support
» Revision
74
: hooks/charmhelpers/fetch/__init__.py
« back to all changes in this revision
Viewing changes to hooks/charmhelpers/fetch/__init__.py
- browse files at revision 74
- compare with another revision
- download diff
- download tarball
- view history from revision 74
- Committer: Jorge Niedbalski
- Date: 2016-09-20 18:01:20 UTC
- mfrom: (72.2.3 memcached.py3)
- Revision ID: jorge.niedbalski@canonical.com-20160920180120-apb4ut3cugwxcrer
[freyes, r=niedbalski] Fixes bug LP: #1576458
- files added:
- .coveragerc
- hooks/charmhelpers/core/host_factory
- hooks/charmhelpers/core/kernel_factory
- files modified:
- .bzrignore
added
removed
hooks/charmhelpers/fetch/__init__.py
1
1
# Copyright 2014-2015 Canonical Limited.
2
2
#
3
# This file is part of charm-helpers.
4
#
5
# charm-helpers is free software: you can redistribute it and/or modify
6
# it under the terms of the GNU Lesser General Public License version 3 as
7
# published by the Free Software Foundation.
8
#
9
# charm-helpers is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU Lesser General Public License for more details.
13
#
14
# You should have received a copy of the GNU Lesser General Public License
15
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
# http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
16
14
17
15
import importlib
18
from tempfile import NamedTemporaryFile
19
import time
16
from charmhelpers.osplatform import get_platform
20
17
from yaml import safe_load
21
from charmhelpers.core.host import (
22
lsb_release
23
)
24
import subprocess
25
18
from charmhelpers.core.hookenv import (
26
19
config,
27
20
log,
28
21
)
29
import os
30
22
31
23
import six
32
24
if six.PY3:
35
27
from urlparse import urlparse, urlunparse
36
28
37
29
38
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
39
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
40
"""
41
PROPOSED_POCKET = """# Proposed
42
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
43
"""
44
CLOUD_ARCHIVE_POCKETS = {
45
# Folsom
46
'folsom': 'precise-updates/folsom',
47
'precise-folsom': 'precise-updates/folsom',
48
'precise-folsom/updates': 'precise-updates/folsom',
49
'precise-updates/folsom': 'precise-updates/folsom',
50
'folsom/proposed': 'precise-proposed/folsom',
51
'precise-folsom/proposed': 'precise-proposed/folsom',
52
'precise-proposed/folsom': 'precise-proposed/folsom',
53
# Grizzly
54
'grizzly': 'precise-updates/grizzly',
55
'precise-grizzly': 'precise-updates/grizzly',
56
'precise-grizzly/updates': 'precise-updates/grizzly',
57
'precise-updates/grizzly': 'precise-updates/grizzly',
58
'grizzly/proposed': 'precise-proposed/grizzly',
59
'precise-grizzly/proposed': 'precise-proposed/grizzly',
60
'precise-proposed/grizzly': 'precise-proposed/grizzly',
61
# Havana
62
'havana': 'precise-updates/havana',
63
'precise-havana': 'precise-updates/havana',
64
'precise-havana/updates': 'precise-updates/havana',
65
'precise-updates/havana': 'precise-updates/havana',
66
'havana/proposed': 'precise-proposed/havana',
67
'precise-havana/proposed': 'precise-proposed/havana',
68
'precise-proposed/havana': 'precise-proposed/havana',
69
# Icehouse
70
'icehouse': 'precise-updates/icehouse',
71
'precise-icehouse': 'precise-updates/icehouse',
72
'precise-icehouse/updates': 'precise-updates/icehouse',
73
'precise-updates/icehouse': 'precise-updates/icehouse',
74
'icehouse/proposed': 'precise-proposed/icehouse',
75
'precise-icehouse/proposed': 'precise-proposed/icehouse',
76
'precise-proposed/icehouse': 'precise-proposed/icehouse',
77
# Juno
78
'juno': 'trusty-updates/juno',
79
'trusty-juno': 'trusty-updates/juno',
80
'trusty-juno/updates': 'trusty-updates/juno',
81
'trusty-updates/juno': 'trusty-updates/juno',
82
'juno/proposed': 'trusty-proposed/juno',
83
'trusty-juno/proposed': 'trusty-proposed/juno',
84
'trusty-proposed/juno': 'trusty-proposed/juno',
85
# Kilo
86
'kilo': 'trusty-updates/kilo',
87
'trusty-kilo': 'trusty-updates/kilo',
88
'trusty-kilo/updates': 'trusty-updates/kilo',
89
'trusty-updates/kilo': 'trusty-updates/kilo',
90
'kilo/proposed': 'trusty-proposed/kilo',
91
'trusty-kilo/proposed': 'trusty-proposed/kilo',
92
'trusty-proposed/kilo': 'trusty-proposed/kilo',
93
}
94
95
30
# The order of this list is very important. Handlers should be listed in from
96
31
# least- to most-specific URL matching.
97
32
FETCH_HANDLERS = (
100
35
'charmhelpers.fetch.giturl.GitUrlFetchHandler',
101
36
)
102
37
103
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
104
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
105
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
106
107
38
108
39
class SourceConfigError(Exception):
109
40
pass
141
72
return urlunparse(parts)
142
73
143
74
144
def filter_installed_packages(packages):
145
"""Returns a list of packages that require installation"""
146
cache = apt_cache()
147
_pkgs = []
148
for package in packages:
149
try:
150
p = cache[package]
151
p.current_ver or _pkgs.append(package)
152
except KeyError:
153
log('Package {} has no installation candidate.'.format(package),
154
level='WARNING')
155
_pkgs.append(package)
156
return _pkgs
157
158
159
def apt_cache(in_memory=True):
160
"""Build and return an apt cache"""
161
from apt import apt_pkg
162
apt_pkg.init()
163
if in_memory:
164
apt_pkg.config.set("Dir::Cache::pkgcache", "")
165
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
166
return apt_pkg.Cache()
167
168
169
def apt_install(packages, options=None, fatal=False):
170
"""Install one or more packages"""
171
if options is None:
172
options = ['--option=Dpkg::Options::=--force-confold']
173
174
cmd = ['apt-get', '--assume-yes']
175
cmd.extend(options)
176
cmd.append('install')
177
if isinstance(packages, six.string_types):
178
cmd.append(packages)
179
else:
180
cmd.extend(packages)
181
log("Installing {} with options: {}".format(packages,
182
options))
183
_run_apt_command(cmd, fatal)
184
185
186
def apt_upgrade(options=None, fatal=False, dist=False):
187
"""Upgrade all packages"""
188
if options is None:
189
options = ['--option=Dpkg::Options::=--force-confold']
190
191
cmd = ['apt-get', '--assume-yes']
192
cmd.extend(options)
193
if dist:
194
cmd.append('dist-upgrade')
195
else:
196
cmd.append('upgrade')
197
log("Upgrading with options: {}".format(options))
198
_run_apt_command(cmd, fatal)
199
200
201
def apt_update(fatal=False):
202
"""Update local apt cache"""
203
cmd = ['apt-get', 'update']
204
_run_apt_command(cmd, fatal)
205
206
207
def apt_purge(packages, fatal=False):
208
"""Purge one or more packages"""
209
cmd = ['apt-get', '--assume-yes', 'purge']
210
if isinstance(packages, six.string_types):
211
cmd.append(packages)
212
else:
213
cmd.extend(packages)
214
log("Purging {}".format(packages))
215
_run_apt_command(cmd, fatal)
216
217
218
def apt_mark(packages, mark, fatal=False):
219
"""Flag one or more packages using apt-mark"""
220
cmd = ['apt-mark', mark]
221
if isinstance(packages, six.string_types):
222
cmd.append(packages)
223
else:
224
cmd.extend(packages)
225
log("Holding {}".format(packages))
226
227
if fatal:
228
subprocess.check_call(cmd, universal_newlines=True)
229
else:
230
subprocess.call(cmd, universal_newlines=True)
231
232
233
def apt_hold(packages, fatal=False):
234
return apt_mark(packages, 'hold', fatal=fatal)
235
236
237
def apt_unhold(packages, fatal=False):
238
return apt_mark(packages, 'unhold', fatal=fatal)
239
240
241
def add_source(source, key=None):
242
"""Add a package source to this system.
243
244
@param source: a URL or sources.list entry, as supported by
245
add-apt-repository(1). Examples::
246
247
ppa:charmers/example
248
deb https://stub:key@private.example.com/ubuntu trusty main
249
250
In addition:
251
'proposed:' may be used to enable the standard 'proposed'
252
pocket for the release.
253
'cloud:' may be used to activate official cloud archive pockets,
254
such as 'cloud:icehouse'
255
'distro' may be used as a noop
256
257
@param key: A key to be added to the system's APT keyring and used
258
to verify the signatures on packages. Ideally, this should be an
259
ASCII format GPG public key including the block headers. A GPG key
260
id may also be used, but be aware that only insecure protocols are
261
available to retrieve the actual public key from a public keyserver
262
placing your Juju environment at risk. ppa and cloud archive keys
263
are securely added automtically, so sould not be provided.
264
"""
265
if source is None:
266
log('Source is not present. Skipping')
267
return
268
269
if (source.startswith('ppa:') or
270
source.startswith('http') or
271
source.startswith('deb ') or
272
source.startswith('cloud-archive:')):
273
subprocess.check_call(['add-apt-repository', '--yes', source])
274
elif source.startswith('cloud:'):
275
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
276
fatal=True)
277
pocket = source.split(':')[-1]
278
if pocket not in CLOUD_ARCHIVE_POCKETS:
279
raise SourceConfigError(
280
'Unsupported cloud: source option %s' %
281
pocket)
282
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
283
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
284
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
285
elif source == 'proposed':
286
release = lsb_release()['DISTRIB_CODENAME']
287
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
288
apt.write(PROPOSED_POCKET.format(release))
289
elif source == 'distro':
290
pass
291
else:
292
log("Unknown source: {!r}".format(source))
293
294
if key:
295
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
296
with NamedTemporaryFile('w+') as key_file:
297
key_file.write(key)
298
key_file.flush()
299
key_file.seek(0)
300
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
301
else:
302
# Note that hkp: is in no way a secure protocol. Using a
303
# GPG key id is pointless from a security POV unless you
304
# absolutely trust your network and DNS.
305
subprocess.check_call(['apt-key', 'adv', '--keyserver',
306
'hkp://keyserver.ubuntu.com:80', '--recv',
307
key])
75
__platform__ = get_platform()
76
module = "charmhelpers.fetch.%s" % __platform__
77
fetch = importlib.import_module(module)
78
79
filter_installed_packages = fetch.filter_installed_packages
80
install = fetch.install
81
upgrade = fetch.upgrade
82
update = fetch.update
83
purge = fetch.purge
84
add_source = fetch.add_source
85
86
if __platform__ == "ubuntu":
87
apt_cache = fetch.apt_cache
88
apt_install = fetch.install
89
apt_update = fetch.update
90
apt_upgrade = fetch.upgrade
91
apt_purge = fetch.purge
92
apt_mark = fetch.apt_mark
93
apt_hold = fetch.apt_hold
94
apt_unhold = fetch.apt_unhold
95
elif __platform__ == "centos":
96
yum_search = fetch.yum_search
308
97
309
98
310
99
def configure_sources(update=False,
311
100
sources_var='install_sources',
312
101
keys_var='install_keys'):
313
"""
314
Configure multiple sources from charm configuration.
102
"""Configure multiple sources from charm configuration.
315
103
316
104
The lists are encoded as yaml fragments in the configuration.
317
The frament needs to be included as a string. Sources and their
105
The fragment needs to be included as a string. Sources and their
318
106
corresponding keys are of the types supported by add_source().
319
107
320
108
Example config:
346
134
for source, key in zip(sources, keys):
347
135
add_source(source, key)
348
136
if update:
349
apt_update(fatal=True)
137
fetch.update(fatal=True)
350
138
351
139
352
140
def install_remote(source, *args, **kwargs):
353
"""
354
Install a file tree from a remote source
141
"""Install a file tree from a remote source.
355
142
356
143
The specified source should be a url of the form:
357
144
scheme://[host]/path[#[option=value][&...]]
374
161
# We ONLY check for True here because can_handle may return a string
375
162
# explaining why it can't handle a given source.
376
163
handlers = [h for h in plugins() if h.can_handle(source) is True]
377
installed_to = None
378
164
for handler in handlers:
379
165
try:
380
installed_to = handler.install(source, *args, **kwargs)
381
except UnhandledSource:
382
pass
383
if not installed_to:
384
raise UnhandledSource("No handler found for source {}".format(source))
385
return installed_to
166
return handler.install(source, *args, **kwargs)
167
except UnhandledSource as e:
168
log('Install source attempt unsuccessful: {}'.format(e),
169
level='WARNING')
170
raise UnhandledSource("No handler found for source {}".format(source))
386
171
387
172
388
173
def install_from_config(config_var_name):
174
"""Install a file from config."""
389
175
charm_config = config()
390
176
source = charm_config[config_var_name]
391
177
return install_remote(source)
402
188
importlib.import_module(package),
403
189
classname)
404
190
plugin_list.append(handler_class())
405
except (ImportError, AttributeError):
191
except NotImplementedError:
406
192
# Skip missing plugins so that they can be ommitted from
407
193
# installation if desired
408
194
log("FetchHandler {} not found, skipping plugin".format(
409
195
handler_name))
410
196
return plugin_list
411
412
413
def _run_apt_command(cmd, fatal=False):
414
"""
415
Run an APT command, checking output and retrying if the fatal flag is set
416
to True.
417
418
:param: cmd: str: The apt command to run.
419
:param: fatal: bool: Whether the command's output should be checked and
420
retried.
421
"""
422
env = os.environ.copy()
423
424
if 'DEBIAN_FRONTEND' not in env:
425
env['DEBIAN_FRONTEND'] = 'noninteractive'
426
427
if fatal:
428
retry_count = 0
429
result = None
430
431
# If the command is considered "fatal", we need to retry if the apt
432
# lock was not acquired.
433
434
while result is None or result == APT_NO_LOCK:
435
try:
436
result = subprocess.check_call(cmd, env=env)
437
except subprocess.CalledProcessError as e:
438
retry_count = retry_count + 1
439
if retry_count > APT_NO_LOCK_RETRY_COUNT:
440
raise
441
result = e.returncode
442
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
443
"".format(APT_NO_LOCK_RETRY_DELAY))
444
time.sleep(APT_NO_LOCK_RETRY_DELAY)
445
446
else:
447
subprocess.call(cmd, env=env)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1