~andrewjbeach/juju-ci-tools/make-local-patcher
» Revision
250.1.2
(compared to revision 1485.3.16)
: jujupy.py
« back to all changes in this revision
Viewing changes to jujupy.py
- browse files at revision 250.1.2
- compare with another revision
- download diff
- download tarball
- view history from revision 250.1.2
- Committer: Aaron Bentley
- Date: 2014-02-24 17:18:29 UTC
- mto: This revision was merged to the branch mainline in revision 252.
- Revision ID: aaron.bentley@canonical.com-20140224171829-sz644yhoygu7m9dm
Use tags to identify and shut down instances.
- files added:
- assemble-public-tools.bash
- azure
- azure-sdk-for-python-master
- azure-sdk-for-python-master/azure
- azure-sdk-for-python-master/azure/http
- azure-sdk-for-python-master/azure/servicebus
- azure-sdk-for-python-master/azure/servicemanagement
- azure-sdk-for-python-master/azure/storage
- files removed:
- Makefile
- maas
- maas-p8
- perf_static
- tests
- files renamed:
- assess_win_client.py => test-win-client
- tests/test_jujupy.py => test_jujupy.py
- files modified:
- .bzrignore
added
removed
jujupy.py
1
1
from __future__ import print_function
2
2
3
from collections import (
4
defaultdict,
5
namedtuple,
6
)
7
from contextlib import (
8
contextmanager,
9
nested,
10
)
11
from copy import deepcopy
3
__metaclass__ = type
4
5
import yaml
6
7
from collections import defaultdict
12
8
from cStringIO import StringIO
13
from datetime import datetime
14
import errno
15
from itertools import chain
16
import json
17
import logging
9
from datetime import datetime, timedelta
10
import httplib
18
11
import os
19
import pexpect
20
import re
21
from shutil import rmtree
12
import socket
22
13
import subprocess
23
14
import sys
24
from tempfile import NamedTemporaryFile
25
import time
26
27
import yaml
28
29
from jujuconfig import (
30
get_environments_path,
31
get_jenv_path,
32
get_juju_home,
33
get_selected_environment,
34
)
35
from utility import (
36
check_free_disk_space,
37
ensure_deleted,
38
ensure_dir,
39
is_ipv6_address,
40
JujuResourceTimeout,
41
pause,
42
quote,
43
qualified_model_name,
44
scoped_environ,
45
split_address_port,
46
temp_dir,
47
unqualified_model_name,
48
until_timeout,
49
)
50
51
52
__metaclass__ = type
53
54
AGENTS_READY = set(['started', 'idle'])
15
import tempfile
16
from time import sleep
17
import urllib2
18
19
from jujuconfig import get_selected_environment
20
21
55
22
WIN_JUJU_CMD = os.path.join('\\', 'Progra~2', 'Juju', 'juju.exe')
56
23
57
JUJU_DEV_FEATURE_FLAGS = 'JUJU_DEV_FEATURE_FLAGS'
58
CONTROLLER = 'controller'
59
KILL_CONTROLLER = 'kill-controller'
60
SYSTEM = 'system'
61
62
KVM_MACHINE = 'kvm'
63
LXC_MACHINE = 'lxc'
64
LXD_MACHINE = 'lxd'
65
66
_DEFAULT_BUNDLE_TIMEOUT = 3600
67
68
_jes_cmds = {KILL_CONTROLLER: {
69
'create': 'create-environment',
70
'kill': KILL_CONTROLLER,
71
}}
72
for super_cmd in [SYSTEM, CONTROLLER]:
73
_jes_cmds[super_cmd] = {
74
'create': '{} create-environment'.format(super_cmd),
75
'kill': '{} kill'.format(super_cmd),
76
}
77
78
log = logging.getLogger("jujupy")
79
80
81
class IncompatibleConfigClass(Exception):
82
"""Raised when a client is initialised with the wrong config class."""
83
84
85
class SoftDeadlineExceeded(Exception):
86
"""Raised when an overall client operation takes too long."""
87
88
89
def get_timeout_path():
90
import timeout
91
return os.path.abspath(timeout.__file__)
92
93
94
def get_timeout_prefix(duration, timeout_path=None):
95
"""Return extra arguments to run a command with a timeout."""
96
if timeout_path is None:
97
timeout_path = get_timeout_path()
98
return (sys.executable, timeout_path, '%.2f' % duration, '--')
99
100
101
def get_teardown_timeout(client):
102
"""Return the timeout need byt the client to teardown resources."""
103
if client.env.config['type'] == 'azure':
104
return 1800
105
else:
106
return 600
107
108
109
def parse_new_state_server_from_error(error):
110
err_str = str(error)
111
output = getattr(error, 'output', None)
112
if output is not None:
113
err_str += output
114
matches = re.findall(r'Attempting to connect to (.*):22', err_str)
115
if matches:
116
return matches[-1]
117
return None
118
119
24
120
25
class ErroredUnit(Exception):
121
26
122
def __init__(self, unit_name, state):
123
msg = '%s is in state %s' % (unit_name, state)
27
def __init__(self, environment, unit_name, state):
28
msg = '<%s> %s is in state %s' % (environment, unit_name, state)
124
29
Exception.__init__(self, msg)
125
self.unit_name = unit_name
126
self.state = state
127
128
129
class BootstrapMismatch(Exception):
130
131
def __init__(self, arg_name, arg_val, env_name, env_val):
132
super(BootstrapMismatch, self).__init__(
133
'--{} {} does not match {}: {}'.format(
134
arg_name, arg_val, env_name, env_val))
135
136
137
class UpgradeMongoNotSupported(Exception):
138
139
def __init__(self):
140
super(UpgradeMongoNotSupported, self).__init__(
141
'This client does not support upgrade-mongo')
142
143
144
class JESNotSupported(Exception):
145
146
def __init__(self):
147
super(JESNotSupported, self).__init__(
148
'This client does not support JES')
149
150
151
class JESByDefault(Exception):
152
153
def __init__(self):
154
super(JESByDefault, self).__init__(
155
'This client does not need to enable JES')
156
157
158
Machine = namedtuple('Machine', ['machine_id', 'info'])
30
31
32
class until_timeout:
33
34
"""Yields None until timeout is reached.
35
36
:ivar timeout: Number of seconds to wait.
37
"""
38
def __init__(self, timeout):
39
self.timeout = timeout
40
self.start = self.now()
41
42
def __iter__(self):
43
return self
44
45
@staticmethod
46
def now():
47
return datetime.now()
48
49
def next(self):
50
if self.now() - self.start >= timedelta(0, self.timeout):
51
raise StopIteration
52
return None
159
53
160
54
161
55
def yaml_loads(yaml_str):
162
56
return yaml.safe_load(StringIO(yaml_str))
163
57
164
58
165
def coalesce_agent_status(agent_item):
166
"""Return the machine agent-state or the unit agent-status."""
167
state = agent_item.get('agent-state')
168
if state is None and agent_item.get('agent-status') is not None:
169
state = agent_item.get('agent-status').get('current')
170
if state is None and agent_item.get('juju-status') is not None:
171
state = agent_item.get('juju-status').get('current')
172
if state is None:
173
state = 'no-agent'
174
return state
175
176
177
59
class CannotConnectEnv(subprocess.CalledProcessError):
178
60
179
61
def __init__(self, e):
180
62
super(CannotConnectEnv, self).__init__(e.returncode, e.cmd, e.output)
181
63
182
64
183
class StatusNotMet(Exception):
184
185
_fmt = 'Expected status not reached in {env}.'
186
187
def __init__(self, environment_name, status):
188
self.env = environment_name
189
self.status = status
190
191
def __str__(self):
192
return self._fmt.format(env=self.env)
193
194
195
class AgentsNotStarted(StatusNotMet):
196
197
_fmt = 'Timed out waiting for agents to start in {env}.'
198
199
200
class VersionsNotUpdated(StatusNotMet):
201
202
_fmt = 'Some versions did not update.'
203
204
205
class WorkloadsNotReady(StatusNotMet):
206
207
_fmt = 'Workloads not ready in {env}.'
208
209
210
@contextmanager
211
def temp_yaml_file(yaml_dict):
212
temp_file = NamedTemporaryFile(suffix='.yaml', delete=False)
213
try:
214
with temp_file:
215
yaml.safe_dump(yaml_dict, temp_file)
216
yield temp_file.name
217
finally:
218
os.unlink(temp_file.name)
219
220
221
class SimpleEnvironment:
222
"""Represents a model in a JUJU_HOME directory for juju 1."""
223
224
def __init__(self, environment, config=None, juju_home=None,
225
controller=None):
226
"""Constructor.
227
228
:param environment: Name of the environment.
229
:param config: Dictionary with configuration options, default is None.
230
:param juju_home: Path to JUJU_HOME directory, default is None.
231
:param controller: Controller instance-- this model's controller.
232
If not given or None a new instance is created."""
233
self.user_name = None
234
if controller is None:
235
controller = Controller(environment)
236
self.controller = controller
237
self.environment = environment
238
self.config = config
239
self.juju_home = juju_home
240
if self.config is not None:
241
self.local = bool(self.config.get('type') == 'local')
242
self.kvm = (
243
self.local and bool(self.config.get('container') == 'kvm'))
244
self.maas = bool(self.config.get('type') == 'maas')
245
self.joyent = bool(self.config.get('type') == 'joyent')
246
else:
247
self.local = False
248
self.kvm = False
249
self.maas = False
250
self.joyent = False
251
252
def clone(self, model_name=None):
253
config = deepcopy(self.config)
254
if model_name is None:
255
model_name = self.environment
256
else:
257
config['name'] = unqualified_model_name(model_name)
258
result = self.__class__(model_name, config, self.juju_home,
259
self.controller)
260
result.local = self.local
261
result.kvm = self.kvm
262
result.maas = self.maas
263
result.joyent = self.joyent
264
return result
265
266
def __eq__(self, other):
267
if type(self) != type(other):
268
return False
269
if self.environment != other.environment:
270
return False
271
if self.config != other.config:
272
return False
273
if self.local != other.local:
274
return False
275
if self.maas != other.maas:
276
return False
277
return True
278
279
def __ne__(self, other):
280
return not self == other
281
282
def set_model_name(self, model_name, set_controller=True):
283
if set_controller:
284
self.controller.name = model_name
285
self.environment = model_name
286
self.config['name'] = unqualified_model_name(model_name)
287
288
@classmethod
289
def from_config(cls, name):
290
"""Create an environment from the configuation file.
291
292
:param name: Name of the environment to get the configuration from."""
293
return cls._from_config(name)
294
295
@classmethod
296
def _from_config(cls, name):
297
config, selected = get_selected_environment(name)
298
if name is None:
299
name = selected
300
return cls(name, config)
301
302
def needs_sudo(self):
303
return self.local
304
305
@contextmanager
306
def make_jes_home(self, juju_home, dir_name, new_config):
307
home_path = jes_home_path(juju_home, dir_name)
308
if os.path.exists(home_path):
309
rmtree(home_path)
310
os.makedirs(home_path)
311
self.dump_yaml(home_path, new_config)
312
yield home_path
313
314
def get_cloud_credentials(self):
315
"""Return the credentials for this model's cloud.
316
317
This implementation returns config variables in addition to
318
credentials.
319
"""
320
return self.config
321
322
def dump_yaml(self, path, config):
323
dump_environments_yaml(path, config)
324
325
326
class JujuData(SimpleEnvironment):
327
"""Represents a model in a JUJU_DATA directory for juju 2."""
328
329
def __init__(self, environment, config=None, juju_home=None,
330
controller=None):
331
"""Constructor.
332
333
This extends SimpleEnvironment's constructor.
334
335
:param environment: Name of the environment.
336
:param config: Dictionary with configuration options; default is None.
337
:param juju_home: Path to JUJU_DATA directory. If None (the default),
338
the home directory is autodetected.
339
:param controller: Controller instance-- this model's controller.
340
If not given or None, a new instance is created.
341
"""
342
if juju_home is None:
343
juju_home = get_juju_home()
344
super(JujuData, self).__init__(environment, config, juju_home,
345
controller)
346
self.credentials = {}
347
self.clouds = {}
348
349
def clone(self, model_name=None):
350
result = super(JujuData, self).clone(model_name)
351
result.credentials = deepcopy(self.credentials)
352
result.clouds = deepcopy(self.clouds)
353
return result
354
355
@classmethod
356
def from_env(cls, env):
357
juju_data = cls(env.environment, env.config, env.juju_home)
358
juju_data.load_yaml()
359
return juju_data
360
361
def load_yaml(self):
362
try:
363
with open(os.path.join(self.juju_home, 'credentials.yaml')) as f:
364
self.credentials = yaml.safe_load(f)
365
except IOError as e:
366
if e.errno != errno.ENOENT:
367
raise RuntimeError(
368
'Failed to read credentials file: {}'.format(str(e)))
369
self.credentials = {}
370
try:
371
with open(os.path.join(self.juju_home, 'clouds.yaml')) as f:
372
self.clouds = yaml.safe_load(f)
373
except IOError as e:
374
if e.errno != errno.ENOENT:
375
raise RuntimeError(
376
'Failed to read clouds file: {}'.format(str(e)))
377
# Default to an empty clouds file.
378
self.clouds = {'clouds': {}}
379
380
@classmethod
381
def from_config(cls, name):
382
"""Create a model from the three configuration files."""
383
juju_data = cls._from_config(name)
384
juju_data.load_yaml()
385
return juju_data
386
387
def dump_yaml(self, path, config):
388
"""Dump the configuration files to the specified path.
389
390
config is unused, but is accepted for compatibility with
391
SimpleEnvironment and make_jes_home().
392
"""
393
with open(os.path.join(path, 'credentials.yaml'), 'w') as f:
394
yaml.safe_dump(self.credentials, f)
395
with open(os.path.join(path, 'clouds.yaml'), 'w') as f:
396
yaml.safe_dump(self.clouds, f)
397
398
def find_endpoint_cloud(self, cloud_type, endpoint):
399
for cloud, cloud_config in self.clouds['clouds'].items():
400
if cloud_config['type'] != cloud_type:
401
continue
402
if cloud_config['endpoint'] == endpoint:
403
return cloud
404
raise LookupError('No such endpoint: {}'.format(endpoint))
405
406
def get_cloud(self):
407
provider = self.config['type']
408
# Separate cloud recommended by: Juju Cloud / Credentials / BootStrap /
409
# Model CLI specification
410
if provider == 'ec2' and self.config['region'] == 'cn-north-1':
411
return 'aws-china'
412
if provider not in ('maas', 'openstack'):
413
return {
414
'ec2': 'aws',
415
'gce': 'google',
416
}.get(provider, provider)
417
if provider == 'maas':
418
endpoint = self.config['maas-server']
419
elif provider == 'openstack':
420
endpoint = self.config['auth-url']
421
return self.find_endpoint_cloud(provider, endpoint)
422
423
def get_region(self):
424
provider = self.config['type']
425
if provider == 'azure':
426
if 'tenant-id' not in self.config:
427
return self.config['location'].replace(' ', '').lower()
428
return self.config['location']
429
elif provider == 'joyent':
430
matcher = re.compile('https://(.*).api.joyentcloud.com')
431
return matcher.match(self.config['sdc-url']).group(1)
432
elif provider == 'lxd':
433
return 'localhost'
434
elif provider == 'manual':
435
return self.config['bootstrap-host']
436
elif provider in ('maas', 'manual'):
437
return None
438
else:
439
return self.config['region']
440
441
def get_cloud_credentials(self):
442
"""Return the credentials for this model's cloud."""
443
cloud_name = self.get_cloud()
444
cloud = self.credentials['credentials'][cloud_name]
445
(credentials,) = cloud.values()
446
return credentials
65
class JujuClientDevel:
66
# This client is meant to work with the latest version of juju.
67
# Subclasses will retain support for older versions of juju, so that the
68
# latest version is easy to read, and older versions can be trivially
69
# deleted.
70
71
def __init__(self, version, full_path):
72
self.version = version
73
self.full_path = full_path
74
75
@classmethod
76
def get_version(cls):
77
return subprocess.check_output(('juju', '--version')).strip()
78
79
@classmethod
80
def get_full_path(cls):
81
if sys.platform == 'win32':
82
return WIN_JUJU_CMD
83
return subprocess.check_output(('which', 'juju')).rstrip('\n')
84
85
@classmethod
86
def by_version(cls):
87
version = cls.get_version()
88
full_path = cls.get_full_path()
89
if version.startswith('1.16'):
90
return JujuClient16(version, full_path)
91
else:
92
return JujuClientDevel(version, full_path)
93
94
def _full_args(self, environment, command, sudo, args):
95
# sudo is not needed for devel releases.
96
e_arg = () if environment is None else ('-e', environment.environment)
97
return ('juju', '--show-log', command,) + e_arg + args
98
99
def bootstrap(self, environment):
100
"""Bootstrap, using sudo if necessary."""
101
if environment.hpcloud:
102
constraints = 'mem=4G'
103
else:
104
constraints = 'mem=2G'
105
self.juju(environment, 'bootstrap', ('--constraints', constraints),
106
environment.needs_sudo())
107
108
def destroy_environment(self, environment):
109
self.juju(
110
None, 'destroy-environment',
111
(environment.environment, '--force', '-y'),
112
environment.needs_sudo(), check=False)
113
114
def get_juju_output(self, environment, command):
115
args = self._full_args(environment, command, False, ())
116
with tempfile.TemporaryFile() as stderr:
117
try:
118
return subprocess.check_output(args, stderr=stderr)
119
except subprocess.CalledProcessError as e:
120
stderr.seek(0)
121
e.stderr = stderr.read()
122
if ('Unable to connect to environment' in e.stderr
123
or 'MissingOrIncorrectVersionHeader' in e.stderr
124
or '307: Temporary Redirect' in e.stderr):
125
raise CannotConnectEnv(e)
126
print('!!! ' + e.stderr)
127
raise
128
129
def get_status(self, environment):
130
"""Get the current status as a dict."""
131
return Status(yaml_loads(self.get_juju_output(environment, 'status')))
132
133
def juju(self, environment, command, args, sudo=False, check=True):
134
"""Run a command under juju for the current environment."""
135
args = self._full_args(environment, command, sudo, args)
136
print(' '.join(args))
137
sys.stdout.flush()
138
if check:
139
return subprocess.check_call(args)
140
return subprocess.call(args)
141
142
143
class JujuClient16(JujuClientDevel):
144
145
def destroy_environment(self, environment):
146
self.juju(environment, 'destroy-environment', ('-y',),
147
environment.needs_sudo(), check=False)
148
149
def _full_args(self, environment, command, sudo, args):
150
# juju 1.16.x required sudo, so replace the juju command with it, as
151
# appropriate.
152
full = super(JujuClient16, self)._full_args(
153
environment, command, sudo, args)
154
sudo_args = ('sudo', '-E', self.full_path) if sudo else ('juju',)
155
return sudo_args + full[1:]
447
156
448
157
449
158
class Status:
450
159
451
def __init__(self, status, status_text):
160
def __init__(self, status):
452
161
self.status = status
453
self.status_text = status_text
454
455
@classmethod
456
def from_text(cls, text):
457
try:
458
# Parsing as JSON is much faster than parsing as YAML, so try
459
# parsing as JSON first and fall back to YAML.
460
status_yaml = json.loads(text)
461
except ValueError:
462
status_yaml = yaml_loads(text)
463
return cls(status_yaml, text)
464
465
def get_applications(self):
466
return self.status.get('applications', {})
467
468
def iter_machines(self, containers=False, machines=True):
162
163
def agent_items(self):
469
164
for machine_name, machine in sorted(self.status['machines'].items()):
470
if machines:
471
yield machine_name, machine
472
if containers:
473
for contained, unit in machine.get('containers', {}).items():
474
yield contained, unit
475
476
def iter_new_machines(self, old_status):
477
for machine, data in self.iter_machines():
478
if machine in old_status.status['machines']:
479
continue
480
yield machine, data
481
482
def iter_units(self):
483
for service_name, service in sorted(self.get_applications().items()):
484
for unit_name, unit in sorted(service.get('units', {}).items()):
165
yield machine_name, machine
166
for service in sorted(self.status['services'].values()):
167
for unit_name, unit in service.get('units', {}).items():
485
168
yield unit_name, unit
486
subordinates = unit.get('subordinates', ())
487
for sub_name in sorted(subordinates):
488
yield sub_name, subordinates[sub_name]
489
490
def agent_items(self):
491
for machine_name, machine in self.iter_machines(containers=True):
492
yield machine_name, machine
493
for unit_name, unit in self.iter_units():
494
yield unit_name, unit
495
169
496
170
def agent_states(self):
497
171
"""Map agent states to the units and machines in those states."""
498
172
states = defaultdict(list)
499
173
for item_name, item in self.agent_items():
500
states[coalesce_agent_status(item)].append(item_name)
174
states[item.get('agent-state', 'no-agent')].append(item_name)
501
175
return states
502
176
503
def check_agents_started(self, environment_name=None):
177
def check_agents_started(self, environment_name):
504
178
"""Check whether all agents are in the 'started' state.
505
179
506
180
If not, return agent_states output. If so, return None.
507
181
If an error is encountered for an agent, raise ErroredUnit
508
182
"""
509
bad_state_info = re.compile(
510
'(.*error|^(cannot set up groups|cannot run instance)).*')
183
# Look for errors preventing an agent from being installed
511
184
for item_name, item in self.agent_items():
512
185
state_info = item.get('agent-state-info', '')
513
if bad_state_info.match(state_info):
514
raise ErroredUnit(item_name, state_info)
186
if 'error' in state_info:
187
raise ErroredUnit(environment_name, item_name, state_info)
515
188
states = self.agent_states()
516
if set(states.keys()).issubset(AGENTS_READY):
189
if states.keys() == ['started']:
517
190
return None
518
191
for state, entries in states.items():
519
192
if 'error' in state:
520
# sometimes the state may be hidden in juju status message
521
juju_status = dict(
522
self.agent_items())[entries[0]].get('juju-status')
523
if juju_status:
524
juju_status_msg = juju_status.get('message')
525
if juju_status_msg:
526
state = juju_status_msg
527
raise ErroredUnit(entries[0], state)
193
raise ErroredUnit(environment_name, entries[0], state)
528
194
return states
529
195
530
def get_service_count(self):
531
return len(self.get_applications())
532
533
def get_service_unit_count(self, service):
534
return len(
535
self.get_applications().get(service, {}).get('units', {}))
536
537
196
def get_agent_versions(self):
538
197
versions = defaultdict(set)
539
198
for item_name, item in self.agent_items():
540
if item.get('juju-status', None):
541
version = item['juju-status'].get('version', 'unknown')
542
versions[version].add(item_name)
543
else:
544
versions[item.get('agent-version', 'unknown')].add(item_name)
199
versions[item.get('agent-version', 'unknown')].add(item_name)
545
200
return versions
546
201
547
def get_instance_id(self, machine_id):
548
return self.status['machines'][machine_id]['instance-id']
549
550
def get_unit(self, unit_name):
551
"""Return metadata about a unit."""
552
for service in sorted(self.get_applications().values()):
553
if unit_name in service.get('units', {}):
554
return service['units'][unit_name]
555
raise KeyError(unit_name)
556
557
def service_subordinate_units(self, service_name):
558
"""Return subordinate metadata for a service_name."""
559
services = self.get_applications()
560
if service_name in services:
561
for unit in sorted(services[service_name].get(
562
'units', {}).values()):
563
for sub_name, sub in unit.get('subordinates', {}).items():
564
yield sub_name, sub
565
566
def get_open_ports(self, unit_name):
567
"""List the open ports for the specified unit.
568
569
If no ports are listed for the unit, the empty list is returned.
570
"""
571
return self.get_unit(unit_name).get('open-ports', [])
572
573
574
class ServiceStatus(Status):
575
576
def get_applications(self):
577
return self.status.get('services', {})
578
579
580
class Juju2Backend:
581
"""A Juju backend referring to a specific juju 2 binary.
582
583
Uses -m to specify models, uses JUJU_DATA to specify home directory.
584
"""
585
586
def __init__(self, full_path, version, feature_flags, debug,
587
soft_deadline=None):
588
self._version = version
589
self._full_path = full_path
590
self.feature_flags = feature_flags
591
self.debug = debug
592
self._timeout_path = get_timeout_path()
593
self.juju_timings = {}
594
self.soft_deadline = soft_deadline
595
self._ignore_soft_deadline = False
596
597
def _now(self):
598
return datetime.utcnow()
599
600
@contextmanager
601
def _check_timeouts(self):
602
# If an exception occurred, we don't want to replace it with
603
# SoftDeadlineExceeded.
604
yield
605
if self.soft_deadline is None or self._ignore_soft_deadline:
606
return
607
if self._now() > self.soft_deadline:
608
raise SoftDeadlineExceeded('Operation exceeded deadline.')
609
610
@contextmanager
611
def ignore_soft_deadline(self):
612
"""Ignore the client deadline. For cleanup code."""
613
old_val = self._ignore_soft_deadline
614
self._ignore_soft_deadline = True
615
try:
616
yield
617
finally:
618
self._ignore_soft_deadline = old_val
619
620
def clone(self, full_path, version, debug, feature_flags):
621
if version is None:
622
version = self.version
623
if full_path is None:
624
full_path = self.full_path
625
if debug is None:
626
debug = self.debug
627
result = self.__class__(full_path, version, feature_flags, debug,
628
self.soft_deadline)
629
return result
630
631
@property
632
def version(self):
633
return self._version
634
635
@property
636
def full_path(self):
637
return self._full_path
638
639
@property
640
def juju_name(self):
641
return os.path.basename(self._full_path)
642
643
def _get_attr_tuple(self):
644
return (self._version, self._full_path, self.feature_flags,
645
self.debug, self.juju_timings)
646
647
def __eq__(self, other):
648
if type(self) != type(other):
649
return False
650
return self._get_attr_tuple() == other._get_attr_tuple()
651
652
def shell_environ(self, used_feature_flags, juju_home):
653
"""Generate a suitable shell environment.
654
655
Juju's directory must be in the PATH to support plugins.
656
"""
657
env = dict(os.environ)
658
if self.full_path is not None:
659
env['PATH'] = '{}{}{}'.format(os.path.dirname(self.full_path),
660
os.pathsep, env['PATH'])
661
flags = self.feature_flags.intersection(used_feature_flags)
662
if flags:
663
env[JUJU_DEV_FEATURE_FLAGS] = ','.join(sorted(flags))
664
env['JUJU_DATA'] = juju_home
665
return env
666
667
def full_args(self, command, args, model, timeout):
668
if model is not None:
669
e_arg = ('-m', model)
670
else:
671
e_arg = ()
672
if timeout is None:
673
prefix = ()
674
else:
675
prefix = get_timeout_prefix(timeout, self._timeout_path)
676
logging = '--debug' if self.debug else '--show-log'
677
678
# If args is a string, make it a tuple. This makes writing commands
679
# with one argument a bit nicer.
680
if isinstance(args, basestring):
681
args = (args,)
682
# we split the command here so that the caller can control where the -m
683
# model flag goes. Everything in the command string is put before the
684
# -m flag.
685
command = command.split()
686
return (prefix + (self.juju_name, logging,) + tuple(command) + e_arg +
687
args)
688
689
def juju(self, command, args, used_feature_flags,
690
juju_home, model=None, check=True, timeout=None, extra_env=None):
691
"""Run a command under juju for the current environment."""
692
args = self.full_args(command, args, model, timeout)
693
log.info(' '.join(args))
694
env = self.shell_environ(used_feature_flags, juju_home)
695
if extra_env is not None:
696
env.update(extra_env)
697
if check:
698
call_func = subprocess.check_call
699
else:
700
call_func = subprocess.call
701
start_time = time.time()
702
# Mutate os.environ instead of supplying env parameter so Windows can
703
# search env['PATH']
704
with scoped_environ(env):
705
with self._check_timeouts():
706
rval = call_func(args)
707
self.juju_timings.setdefault(args, []).append(
708
(time.time() - start_time))
709
return rval
710
711
def expect(self, command, args, used_feature_flags, juju_home, model=None,
712
timeout=None, extra_env=None):
713
args = self.full_args(command, args, model, timeout)
714
log.info(' '.join(args))
715
env = self.shell_environ(used_feature_flags, juju_home)
716
if extra_env is not None:
717
env.update(extra_env)
718
# pexpect.spawn expects a string. This is better than trying to extract
719
# command + args from the returned tuple (as there could be an intial
720
# timing command tacked on).
721
command_string = ' '.join(quote(a) for a in args)
722
with scoped_environ(env):
723
return pexpect.spawn(command_string)
724
725
@contextmanager
726
def juju_async(self, command, args, used_feature_flags,
727
juju_home, model=None, timeout=None):
728
full_args = self.full_args(command, args, model, timeout)
729
log.info(' '.join(args))
730
env = self.shell_environ(used_feature_flags, juju_home)
731
# Mutate os.environ instead of supplying env parameter so Windows can
732
# search env['PATH']
733
with scoped_environ(env):
734
with self._check_timeouts():
735
proc = subprocess.Popen(full_args)
736
yield proc
737
retcode = proc.wait()
738
if retcode != 0:
739
raise subprocess.CalledProcessError(retcode, full_args)
740
741
def get_juju_output(self, command, args, used_feature_flags, juju_home,
742
model=None, timeout=None, user_name=None,
743
merge_stderr=False):
744
args = self.full_args(command, args, model, timeout)
745
env = self.shell_environ(used_feature_flags, juju_home)
746
log.debug(args)
747
# Mutate os.environ instead of supplying env parameter so
748
# Windows can search env['PATH']
749
with scoped_environ(env):
750
proc = subprocess.Popen(
751
args, stdout=subprocess.PIPE, stdin=subprocess.PIPE,
752
stderr=subprocess.STDOUT if merge_stderr else subprocess.PIPE)
753
with self._check_timeouts():
754
sub_output, sub_error = proc.communicate()
755
log.debug(sub_output)
756
if proc.returncode != 0:
757
log.debug(sub_error)
758
e = subprocess.CalledProcessError(
759
proc.returncode, args, sub_output)
760
e.stderr = sub_error
761
if sub_error and (
762
'Unable to connect to environment' in sub_error or
763
'MissingOrIncorrectVersionHeader' in sub_error or
764
'307: Temporary Redirect' in sub_error):
765
raise CannotConnectEnv(e)
766
raise e
767
return sub_output
768
769
def pause(self, seconds):
770
pause(seconds)
771
772
773
class Juju2A2Backend(Juju2Backend):
774
"""Backend for 2A2.
775
776
Uses -m to specify models, uses JUJU_HOME and JUJU_DATA to specify home
777
directory.
778
"""
779
780
def shell_environ(self, used_feature_flags, juju_home):
781
"""Generate a suitable shell environment.
782
783
For 2.0-alpha2 set both JUJU_HOME and JUJU_DATA.
784
"""
785
env = super(Juju2A2Backend, self).shell_environ(used_feature_flags,
786
juju_home)
787
env['JUJU_HOME'] = juju_home
788
return env
789
790
791
class Juju1XBackend(Juju2A2Backend):
792
"""Backend for Juju 1.x - 2A1.
793
794
Uses -e to specify models ("environments", uses JUJU_HOME to specify home
795
directory.
796
"""
797
798
def shell_environ(self, used_feature_flags, juju_home):
799
"""Generate a suitable shell environment.
800
801
For 2.0-alpha1 and earlier set only JUJU_HOME and not JUJU_DATA.
802
"""
803
env = super(Juju1XBackend, self).shell_environ(used_feature_flags,
804
juju_home)
805
env['JUJU_HOME'] = juju_home
806
del env['JUJU_DATA']
807
return env
808
809
def full_args(self, command, args, model, timeout):
810
if model is None:
811
e_arg = ()
812
else:
813
# In 1.x terminology, "model" is "environment".
814
e_arg = ('-e', model)
815
if timeout is None:
816
prefix = ()
817
else:
818
prefix = get_timeout_prefix(timeout, self._timeout_path)
819
logging = '--debug' if self.debug else '--show-log'
820
821
# If args is a string, make it a tuple. This makes writing commands
822
# with one argument a bit nicer.
823
if isinstance(args, basestring):
824
args = (args,)
825
# we split the command here so that the caller can control where the -e
826
# <env> flag goes. Everything in the command string is put before the
827
# -e flag.
828
command = command.split()
829
return (prefix + (self.juju_name, logging,) + tuple(command) + e_arg +
830
args)
831
832
833
def get_client_class(version):
834
if version.startswith('1.16'):
835
raise Exception('Unsupported juju: %s' % version)
836
elif re.match('^1\.22[.-]', version):
837
client_class = EnvJujuClient22
838
elif re.match('^1\.24[.-]', version):
839
client_class = EnvJujuClient24
840
elif re.match('^1\.25[.-]', version):
841
client_class = EnvJujuClient25
842
elif re.match('^1\.26[.-]', version):
843
client_class = EnvJujuClient26
844
elif re.match('^1\.', version):
845
client_class = EnvJujuClient1X
846
# Ensure alpha/beta number matches precisely
847
elif re.match('^2\.0-alpha1([^\d]|$)', version):
848
client_class = EnvJujuClient2A1
849
elif re.match('^2\.0-alpha2([^\d]|$)', version):
850
client_class = EnvJujuClient2A2
851
elif re.match('^2\.0-(alpha3|beta[12])([^\d]|$)', version):
852
client_class = EnvJujuClient2B2
853
elif re.match('^2\.0-(beta[3-6])([^\d]|$)', version):
854
client_class = EnvJujuClient2B3
855
elif re.match('^2\.0-(beta7)([^\d]|$)', version):
856
client_class = EnvJujuClient2B7
857
elif re.match('^2\.0-beta8([^\d]|$)', version):
858
client_class = EnvJujuClient2B8
859
# between beta 9-14
860
elif re.match('^2\.0-beta(9|1[0-4])([^\d]|$)', version):
861
client_class = EnvJujuClient2B9
862
else:
863
client_class = EnvJujuClient
864
return client_class
865
866
867
def client_from_config(config, juju_path, debug=False, soft_deadline=None):
868
"""Create a client from an environment's configuration.
869
870
:param config: Name of the environment to use the config from.
871
:param juju_path: Path to juju binary the client should wrap.
872
:param debug=False: The debug flag for the client, False by default.
873
:param soft_deadline: A datetime representing the deadline by which
874
normal operations should complete. If None, no deadline is
875
enforced.
876
"""
877
version = EnvJujuClient.get_version(juju_path)
878
client_class = get_client_class(version)
879
env = client_class.config_class.from_config(config)
880
if juju_path is None:
881
full_path = EnvJujuClient.get_full_path()
882
else:
883
full_path = os.path.abspath(juju_path)
884
return client_class(env, version, full_path, debug=debug,
885
soft_deadline=soft_deadline)
886
887
888
class EnvJujuClient:
889
"""Wraps calls to a juju instance, associated with a single model.
890
891
Note: A model is often called an enviroment (Juju 1 legacy).
892
893
This class represents the latest Juju version. Subclasses are used to
894
support older versions (see get_client_class).
895
"""
896
897
# The environments.yaml options that are replaced by bootstrap options.
898
#
899
# As described in bug #1538735, default-series and --bootstrap-series must
900
# match. 'default-series' should be here, but is omitted so that
901
# default-series is always forced to match --bootstrap-series.
902
bootstrap_replaces = frozenset(['agent-version'])
903
904
# What feature flags have existed that CI used.
905
known_feature_flags = frozenset([
906
'actions', 'jes', 'address-allocation', 'cloudsigma', 'migration'])
907
908
# What feature flags are used by this version of the juju client.
909
used_feature_flags = frozenset(['address-allocation', 'migration'])
910
911
destroy_model_command = 'destroy-model'
912
913
supported_container_types = frozenset([KVM_MACHINE, LXC_MACHINE,
914
LXD_MACHINE])
915
916
default_backend = Juju2Backend
917
918
config_class = JujuData
919
920
status_class = Status
921
922
agent_metadata_url = 'agent-metadata-url'
923
924
model_permissions = frozenset(['read', 'write', 'admin'])
925
926
controller_permissions = frozenset(['login', 'addmodel', 'superuser'])
927
928
@classmethod
929
def preferred_container(cls):
930
for container_type in [LXD_MACHINE, LXC_MACHINE]:
931
if container_type in cls.supported_container_types:
932
return container_type
933
934
_show_status = 'show-status'
935
936
@classmethod
937
def get_version(cls, juju_path=None):
938
"""Get the version data from a juju binary.
939
940
:param juju_path: Path to binary. If not given or None, 'juju' is used.
941
"""
942
if juju_path is None:
943
juju_path = 'juju'
944
return subprocess.check_output((juju_path, '--version')).strip()
945
946
def check_timeouts(self):
947
return self._backend._check_timeouts()
948
949
def ignore_soft_deadline(self):
950
return self._backend.ignore_soft_deadline()
951
952
def enable_feature(self, flag):
953
"""Enable juju feature by setting the given flag.
954
955
New versions of juju with the feature enabled by default will silently
956
allow this call, but will not export the environment variable.
957
"""
958
if flag not in self.known_feature_flags:
959
raise ValueError('Unknown feature flag: %r' % (flag,))
960
self.feature_flags.add(flag)
961
962
def get_jes_command(self):
963
"""For Juju 2.0, this is always kill-controller."""
964
return KILL_CONTROLLER
965
966
def is_jes_enabled(self):
967
"""Does the state-server support multiple environments."""
968
try:
969
self.get_jes_command()
970
return True
971
except JESNotSupported:
972
return False
973
974
def enable_jes(self):
975
"""Enable JES if JES is optional.
976
977
Specifically implemented by the clients that optionally support JES.
978
This version raises either JESByDefault or JESNotSupported.
979
980
:raises: JESByDefault when JES is always enabled; Juju has the
981
'destroy-controller' command.
982
:raises: JESNotSupported when JES is not supported; Juju does not have
983
the 'system kill' command when the JES feature flag is set.
984
"""
985
if self.is_jes_enabled():
986
raise JESByDefault()
987
else:
988
raise JESNotSupported()
989
990
@classmethod
991
def get_full_path(cls):
992
if sys.platform == 'win32':
993
return WIN_JUJU_CMD
994
return subprocess.check_output(('which', 'juju')).rstrip('\n')
995
996
def clone_path_cls(self, juju_path):
997
"""Clone using the supplied path to determine the class."""
998
version = self.get_version(juju_path)
999
cls = get_client_class(version)
1000
if juju_path is None:
1001
full_path = self.get_full_path()
1002
else:
1003
full_path = os.path.abspath(juju_path)
1004
return self.clone(version=version, full_path=full_path, cls=cls)
1005
1006
def clone(self, env=None, version=None, full_path=None, debug=None,
1007
cls=None):
1008
"""Create a clone of this EnvJujuClient.
1009
1010
By default, the class, environment, version, full_path, and debug
1011
settings will match the original, but each can be overridden.
1012
"""
1013
if env is None:
1014
env = self.env
1015
if cls is None:
1016
cls = self.__class__
1017
feature_flags = self.feature_flags.intersection(cls.used_feature_flags)
1018
backend = self._backend.clone(full_path, version, debug, feature_flags)
1019
other = cls.from_backend(backend, env)
1020
return other
1021
1022
@classmethod
1023
def from_backend(cls, backend, env):
1024
return cls(env=env, version=backend.version,
1025
full_path=backend.full_path,
1026
debug=backend.debug, _backend=backend)
1027
1028
def get_cache_path(self):
1029
return get_cache_path(self.env.juju_home, models=True)
1030
1031
def _cmd_model(self, include_e, controller):
1032
if controller:
1033
return '{controller}:{model}'.format(
1034
controller=self.env.controller.name,
1035
model=self.get_controller_model_name())
1036
elif self.env is None or not include_e:
1037
return None
1038
else:
1039
return '{controller}:{model}'.format(
1040
controller=self.env.controller.name,
1041
model=self.model_name)
1042
1043
def _full_args(self, command, sudo, args,
1044
timeout=None, include_e=True, controller=False):
1045
model = self._cmd_model(include_e, controller)
1046
# sudo is not needed for devel releases.
1047
return self._backend.full_args(command, args, model, timeout)
1048
1049
@staticmethod
1050
def _get_env(env):
1051
if not isinstance(env, JujuData) and isinstance(env,
1052
SimpleEnvironment):
1053
# FIXME: JujuData should be used from the start.
1054
env = JujuData.from_env(env)
1055
return env
1056
1057
def __init__(self, env, version, full_path, juju_home=None, debug=False,
1058
soft_deadline=None, _backend=None):
1059
"""Create a new juju client.
1060
1061
Required Arguments
1062
:param env: Object representing a model in a data directory.
1063
:param version: Version of juju the client wraps.
1064
:param full_path: Full path to juju binary.
1065
1066
Optional Arguments
1067
:param juju_home: default value for env.juju_home. Will be
1068
autodetected if None (the default).
1069
:param debug: Flag to activate debugging output; False by default.
1070
:param soft_deadline: A datetime representing the deadline by which
1071
normal operations should complete. If None, no deadline is
1072
enforced.
1073
:param _backend: The backend to use for interacting with the client.
1074
If None (the default), self.default_backend will be used.
1075
"""
1076
self.env = self._get_env(env)
1077
if _backend is None:
1078
_backend = self.default_backend(full_path, version, set(), debug,
1079
soft_deadline)
1080
self._backend = _backend
1081
if version != _backend.version:
1082
raise ValueError('Version mismatch: {} {}'.format(
1083
version, _backend.version))
1084
if full_path != _backend.full_path:
1085
raise ValueError('Path mismatch: {} {}'.format(
1086
full_path, _backend.full_path))
1087
if debug is not _backend.debug:
1088
raise ValueError('debug mismatch: {} {}'.format(
1089
debug, _backend.debug))
1090
if env is not None:
1091
if juju_home is None:
1092
if env.juju_home is None:
1093
env.juju_home = get_juju_home()
1094
else:
1095
env.juju_home = juju_home
1096
1097
@property
1098
def version(self):
1099
return self._backend.version
1100
1101
@property
1102
def full_path(self):
1103
return self._backend.full_path
1104
1105
@property
1106
def feature_flags(self):
1107
return self._backend.feature_flags
1108
1109
@feature_flags.setter
1110
def feature_flags(self, feature_flags):
1111
self._backend.feature_flags = feature_flags
1112
1113
@property
1114
def debug(self):
1115
return self._backend.debug
1116
1117
@property
1118
def model_name(self):
1119
return self.env.environment
1120
1121
def _shell_environ(self):
1122
"""Generate a suitable shell environment.
1123
1124
Juju's directory must be in the PATH to support plugins.
1125
"""
1126
return self._backend.shell_environ(self.used_feature_flags,
1127
self.env.juju_home)
1128
1129
def add_ssh_machines(self, machines):
1130
for count, machine in enumerate(machines):
1131
try:
1132
self.juju('add-machine', ('ssh:' + machine,))
1133
except subprocess.CalledProcessError:
1134
if count != 0:
1135
raise
1136
logging.warning('add-machine failed. Will retry.')
1137
pause(30)
1138
self.juju('add-machine', ('ssh:' + machine,))
1139
1140
@staticmethod
1141
def get_cloud_region(cloud, region):
1142
if region is None:
1143
return cloud
1144
return '{}/{}'.format(cloud, region)
1145
1146
def get_bootstrap_args(
1147
self, upload_tools, config_filename, bootstrap_series=None,
1148
credential=None, auto_upgrade=False, metadata_source=None,
1149
to=None, agent_version=None):
1150
"""Return the bootstrap arguments for the substrate."""
1151
if self.env.joyent:
1152
# Only accept kvm packages by requiring >1 cpu core, see lp:1446264
1153
constraints = 'mem=2G cpu-cores=1'
1154
else:
1155
constraints = 'mem=2G'
1156
cloud_region = self.get_cloud_region(self.env.get_cloud(),
1157
self.env.get_region())
1158
args = ['--constraints', constraints, self.env.environment,
1159
cloud_region, '--config', config_filename,
1160
'--default-model', self.env.environment]
1161
if upload_tools:
1162
if agent_version is not None:
1163
raise ValueError(
1164
'agent-version may not be given with upload-tools.')
1165
args.insert(0, '--upload-tools')
1166
else:
1167
if agent_version is None:
1168
agent_version = self.get_matching_agent_version()
1169
args.extend(['--agent-version', agent_version])
1170
if bootstrap_series is not None:
1171
args.extend(['--bootstrap-series', bootstrap_series])
1172
if credential is not None:
1173
args.extend(['--credential', credential])
1174
if metadata_source is not None:
1175
args.extend(['--metadata-source', metadata_source])
1176
if auto_upgrade:
1177
args.append('--auto-upgrade')
1178
if to is not None:
1179
args.extend(['--to', to])
1180
return tuple(args)
1181
1182
def add_model(self, env):
1183
"""Add a model to this model's controller and return its client.
1184
1185
:param env: Class representing the new model/environment."""
1186
model_client = self.clone(env)
1187
with model_client._bootstrap_config() as config_file:
1188
self._add_model(env.environment, config_file)
1189
return model_client
1190
1191
def make_model_config(self):
1192
config_dict = make_safe_config(self)
1193
agent_metadata_url = config_dict.pop('tools-metadata-url', None)
1194
if agent_metadata_url is not None:
1195
config_dict.setdefault('agent-metadata-url', agent_metadata_url)
1196
# Strip unneeded variables.
1197
return dict((k, v) for k, v in config_dict.items() if k not in {
1198
'access-key',
1199
'admin-secret',
1200
'application-id',
1201
'application-password',
1202
'auth-url',
1203
'bootstrap-host',
1204
'client-email',
1205
'client-id',
1206
'control-bucket',
1207
'location',
1208
'maas-oauth',
1209
'maas-server',
1210
'management-certificate',
1211
'management-subscription-id',
1212
'manta-key-id',
1213
'manta-user',
1214
'name',
1215
'password',
1216
'private-key',
1217
'region',
1218
'sdc-key-id',
1219
'sdc-url',
1220
'sdc-user',
1221
'secret-key',
1222
'storage-account-name',
1223
'subscription-id',
1224
'tenant-id',
1225
'tenant-name',
1226
'type',
1227
'username',
1228
})
1229
1230
@contextmanager
1231
def _bootstrap_config(self):
1232
with temp_yaml_file(self.make_model_config()) as config_filename:
1233
yield config_filename
1234
1235
def _check_bootstrap(self):
1236
if self.env.environment != self.env.controller.name:
1237
raise AssertionError(
1238
'Controller and environment names should not vary (yet)')
1239
1240
def update_user_name(self):
1241
self.env.user_name = 'admin@local'
1242
1243
def bootstrap(self, upload_tools=False, bootstrap_series=None,
1244
credential=None, auto_upgrade=False, metadata_source=None,
1245
to=None, agent_version=None):
1246
"""Bootstrap a controller."""
1247
self._check_bootstrap()
1248
with self._bootstrap_config() as config_filename:
1249
args = self.get_bootstrap_args(
1250
upload_tools, config_filename, bootstrap_series, credential,
1251
auto_upgrade, metadata_source, to, agent_version)
1252
self.update_user_name()
1253
self.juju('bootstrap', args, include_e=False)
1254
1255
@contextmanager
1256
def bootstrap_async(self, upload_tools=False, bootstrap_series=None,
1257
auto_upgrade=False, metadata_source=None, to=None):
1258
self._check_bootstrap()
1259
with self._bootstrap_config() as config_filename:
1260
args = self.get_bootstrap_args(
1261
upload_tools, config_filename, bootstrap_series, None,
1262
auto_upgrade, metadata_source, to)
1263
self.update_user_name()
1264
with self.juju_async('bootstrap', args, include_e=False):
1265
yield
1266
log.info('Waiting for bootstrap of {}.'.format(
1267
self.env.environment))
1268
1269
def _add_model(self, model_name, config_file):
1270
self.controller_juju('add-model', (
1271
model_name, '--config', config_file))
1272
1273
def destroy_model(self):
1274
exit_status = self.juju(
1275
'destroy-model', (self.env.environment, '-y',),
1276
include_e=False, timeout=get_teardown_timeout(self))
1277
return exit_status
1278
1279
def kill_controller(self):
1280
"""Kill a controller and its environments."""
1281
seen_cmd = self.get_jes_command()
1282
self.juju(
1283
_jes_cmds[seen_cmd]['kill'], (self.env.controller.name, '-y'),
1284
include_e=False, check=False, timeout=get_teardown_timeout(self))
1285
1286
def get_juju_output(self, command, *args, **kwargs):
1287
"""Call a juju command and return the output.
1288
1289
Sub process will be called as 'juju <command> <args> <kwargs>'. Note
1290
that <command> may be a space delimited list of arguments. The -e
1291
<environment> flag will be placed after <command> and before args.
1292
"""
1293
model = self._cmd_model(kwargs.get('include_e', True),
1294
kwargs.get('controller', False))
1295
pass_kwargs = dict(
1296
(k, kwargs[k]) for k in kwargs if k in ['timeout', 'merge_stderr'])
1297
return self._backend.get_juju_output(
1298
command, args, self.used_feature_flags, self.env.juju_home,
1299
model, user_name=self.env.user_name, **pass_kwargs)
1300
1301
def show_status(self):
1302
"""Print the status to output."""
1303
self.juju(self._show_status, ('--format', 'yaml'))
1304
1305
def get_status(self, timeout=60, raw=False, controller=False, *args):
1306
"""Get the current status as a dict."""
1307
# GZ 2015-12-16: Pass remaining timeout into get_juju_output call.
1308
for ignored in until_timeout(timeout):
1309
try:
1310
if raw:
1311
return self.get_juju_output(self._show_status, *args)
1312
return self.status_class.from_text(
1313
self.get_juju_output(
1314
self._show_status, '--format', 'yaml',
1315
controller=controller))
1316
except subprocess.CalledProcessError:
1317
pass
1318
raise Exception(
1319
'Timed out waiting for juju status to succeed')
1320
1321
@staticmethod
1322
def _dict_as_option_strings(options):
1323
return tuple('{}={}'.format(*item) for item in options.items())
1324
1325
def set_config(self, service, options):
1326
option_strings = self._dict_as_option_strings(options)
1327
self.juju('config', (service,) + option_strings)
1328
1329
def get_config(self, service):
1330
return yaml_loads(self.get_juju_output('config', service))
1331
1332
def get_service_config(self, service, timeout=60):
1333
for ignored in until_timeout(timeout):
1334
try:
1335
return self.get_config(service)
1336
except subprocess.CalledProcessError:
1337
pass
1338
raise Exception(
1339
'Timed out waiting for juju get %s' % (service))
1340
1341
def set_model_constraints(self, constraints):
1342
constraint_strings = self._dict_as_option_strings(constraints)
1343
return self.juju('set-model-constraints', constraint_strings)
1344
1345
def get_model_config(self):
1346
"""Return the value of the environment's configured options."""
1347
return yaml.safe_load(
1348
self.get_juju_output('model-config', '--format', 'yaml'))
1349
1350
def get_env_option(self, option):
1351
"""Return the value of the environment's configured option."""
1352
return self.get_juju_output('model-config', option)
1353
1354
def set_env_option(self, option, value):
1355
"""Set the value of the option in the environment."""
1356
option_value = "%s=%s" % (option, value)
1357
return self.juju('model-config', (option_value,))
1358
1359
def unset_env_option(self, option):
1360
"""Unset the value of the option in the environment."""
1361
return self.juju('model-config', ('--reset', option,))
1362
1363
def get_agent_metadata_url(self):
1364
return self.get_env_option(self.agent_metadata_url)
1365
1366
def set_testing_agent_metadata_url(self):
1367
url = self.get_agent_metadata_url()
1368
if 'testing' not in url:
1369
testing_url = url.replace('/tools', '/testing/tools')
1370
self.set_env_option(self.agent_metadata_url, testing_url)
1371
1372
def juju(self, command, args, sudo=False, check=True, include_e=True,
1373
timeout=None, extra_env=None):
1374
"""Run a command under juju for the current environment."""
1375
model = self._cmd_model(include_e, controller=False)
1376
return self._backend.juju(
1377
command, args, self.used_feature_flags, self.env.juju_home,
1378
model, check, timeout, extra_env)
1379
1380
def expect(self, command, args=(), sudo=False, include_e=True,
1381
timeout=None, extra_env=None):
1382
"""Return a process object that is running an interactive `command`.
1383
1384
The interactive command ability is provided by using pexpect.
1385
1386
:param command: String of the juju command to run.
1387
:param args: Tuple containing arguments for the juju `command`.
1388
:param sudo: Whether to call `command` using sudo.
1389
:param include_e: Boolean regarding supplying the juju environment to
1390
`command`.
1391
:param timeout: A float that, if provided, is the timeout in which the
1392
`command` is run.
1393
1394
:return: A pexpect.spawn object that has been called with `command` and
1395
`args`.
1396
1397
"""
1398
model = self._cmd_model(include_e, controller=False)
1399
return self._backend.expect(
1400
command, args, self.used_feature_flags, self.env.juju_home,
1401
model, timeout, extra_env)
1402
1403
def controller_juju(self, command, args):
1404
args = ('-c', self.env.controller.name) + args
1405
return self.juju(command, args, include_e=False)
1406
1407
def get_juju_timings(self):
1408
stringified_timings = {}
1409
for command, timings in self._backend.juju_timings.items():
1410
stringified_timings[' '.join(command)] = timings
1411
return stringified_timings
1412
1413
def juju_async(self, command, args, include_e=True, timeout=None):
1414
model = self._cmd_model(include_e, controller=False)
1415
return self._backend.juju_async(command, args, self.used_feature_flags,
1416
self.env.juju_home, model, timeout)
1417
1418
def deploy(self, charm, repository=None, to=None, series=None,
1419
service=None, force=False, resource=None,
1420
storage=None, constraints=None):
1421
args = [charm]
1422
if service is not None:
1423
args.extend([service])
1424
if to is not None:
1425
args.extend(['--to', to])
1426
if series is not None:
1427
args.extend(['--series', series])
1428
if force is True:
1429
args.extend(['--force'])
1430
if resource is not None:
1431
args.extend(['--resource', resource])
1432
if storage is not None:
1433
args.extend(['--storage', storage])
1434
if constraints is not None:
1435
args.extend(['--constraints', constraints])
1436
return self.juju('deploy', tuple(args))
1437
1438
def attach(self, service, resource):
1439
args = (service, resource)
1440
return self.juju('attach', args)
1441
1442
def list_resources(self, service_or_unit, details=True):
1443
args = ('--format', 'yaml', service_or_unit)
1444
if details:
1445
args = args + ('--details',)
1446
return yaml_loads(self.get_juju_output('list-resources', *args))
1447
1448
def wait_for_resource(self, resource_id, service_or_unit, timeout=60):
1449
log.info('Waiting for resource. Resource id:{}'.format(resource_id))
1450
with self.check_timeouts():
1451
with self.ignore_soft_deadline():
1452
for _ in until_timeout(timeout):
1453
resources_dict = self.list_resources(service_or_unit)
1454
resources = resources_dict['resources']
1455
for resource in resources:
1456
if resource['expected']['resourceid'] == resource_id:
1457
if (resource['expected']['fingerprint'] ==
1458
resource['unit']['fingerprint']):
1459
return
1460
time.sleep(.1)
1461
raise JujuResourceTimeout(
1462
'Timeout waiting for a resource to be downloaded. '
1463
'ResourceId: {} Service or Unit: {} Timeout: {}'.format(
1464
resource_id, service_or_unit, timeout))
1465
1466
def upgrade_charm(self, service, charm_path=None):
1467
args = (service,)
1468
if charm_path is not None:
1469
args = args + ('--path', charm_path)
1470
self.juju('upgrade-charm', args)
1471
1472
def remove_service(self, service):
1473
self.juju('remove-application', (service,))
1474
1475
@classmethod
1476
def format_bundle(cls, bundle_template):
1477
return bundle_template.format(container=cls.preferred_container())
1478
1479
def deploy_bundle(self, bundle_template, timeout=_DEFAULT_BUNDLE_TIMEOUT):
1480
"""Deploy bundle using native juju 2.0 deploy command."""
1481
bundle = self.format_bundle(bundle_template)
1482
self.juju('deploy', bundle, timeout=timeout)
1483
1484
def deployer(self, bundle_template, name=None, deploy_delay=10,
1485
timeout=3600):
1486
"""Deploy a bundle using deployer."""
1487
bundle = self.format_bundle(bundle_template)
1488
args = (
1489
'--debug',
1490
'--deploy-delay', str(deploy_delay),
1491
'--timeout', str(timeout),
1492
'--config', bundle,
1493
)
1494
if name:
1495
args += (name,)
1496
e_arg = ('-e', '{}:{}'.format(
1497
self.env.controller.name, self.env.environment))
1498
args = e_arg + args
1499
self.juju('deployer', args, self.env.needs_sudo(), include_e=False)
1500
1501
def _get_substrate_constraints(self):
1502
if self.env.joyent:
1503
# Only accept kvm packages by requiring >1 cpu core, see lp:1446264
1504
return 'mem=2G cpu-cores=1'
1505
else:
1506
return 'mem=2G'
1507
1508
def quickstart(self, bundle_template, upload_tools=False):
1509
"""quickstart, using sudo if necessary."""
1510
bundle = self.format_bundle(bundle_template)
1511
constraints = 'mem=2G'
1512
args = ('--constraints', constraints)
1513
if upload_tools:
1514
args = ('--upload-tools',) + args
1515
args = args + ('--no-browser', bundle,)
1516
self.juju('quickstart', args, self.env.needs_sudo(),
1517
extra_env={'JUJU': self.full_path})
1518
1519
def status_until(self, timeout, start=None):
1520
"""Call and yield status until the timeout is reached.
1521
1522
Status will always be yielded once before checking the timeout.
1523
1524
This is intended for implementing things like wait_for_started.
1525
1526
:param timeout: The number of seconds to wait before timing out.
1527
:param start: If supplied, the time to count from when determining
1528
timeout.
1529
"""
1530
with self.check_timeouts():
1531
with self.ignore_soft_deadline():
1532
yield self.get_status()
1533
for remaining in until_timeout(timeout, start=start):
1534
yield self.get_status()
1535
1536
def _wait_for_status(self, reporter, translate, exc_type=StatusNotMet,
1537
timeout=1200, start=None):
1538
"""Wait till status reaches an expected state with pretty reporting.
1539
1540
Always tries to get status at least once. Each status call has an
1541
internal timeout of 60 seconds. This is independent of the timeout for
1542
the whole wait, note this means this function may be overrun.
1543
1544
:param reporter: A GroupReporter instance for output.
1545
:param translate: A callable that takes status to make states dict.
1546
:param exc_type: Optional StatusNotMet subclass to raise on timeout.
1547
:param timeout: Optional number of seconds to wait before timing out.
1548
:param start: Optional time to count from when determining timeout.
1549
"""
1550
status = None
1551
try:
1552
with self.check_timeouts():
1553
with self.ignore_soft_deadline():
1554
for _ in chain([None],
1555
until_timeout(timeout, start=start)):
1556
try:
1557
status = self.get_status()
1558
except CannotConnectEnv:
1559
log.info(
1560
'Suppressing "Unable to connect to'
1561
' environment"')
1562
continue
1563
states = translate(status)
1564
if states is None:
1565
break
1566
reporter.update(states)
1567
else:
1568
if status is not None:
1569
log.error(status.status_text)
1570
raise exc_type(self.env.environment, status)
1571
finally:
1572
reporter.finish()
202
203
class Environment:
204
205
def __init__(self, environment, client=None, config=None):
206
self.environment = environment
207
self.client = client
208
self.config = config
209
if self.config is not None:
210
self.local = bool(self.config.get('type') == 'local')
211
self.hpcloud = bool(
212
'hpcloudsvc' in self.config.get('auth-url', ''))
213
else:
214
self.local = False
215
self.hpcloud = False
216
217
@classmethod
218
def from_config(cls, name):
219
client = JujuClientDevel.by_version()
220
return cls(name, client, get_selected_environment(name)[0])
221
222
def needs_sudo(self):
223
return self.local
224
225
def bootstrap(self):
226
return self.client.bootstrap(self)
227
228
def upgrade_juju(self):
229
args = ('--version', self.get_matching_agent_version(no_build=True))
230
if self.local:
231
args += ('--upload-tools',)
232
self.client.juju(self, 'upgrade-juju', args)
233
234
def destroy_environment(self):
235
return self.client.destroy_environment(self)
236
237
def juju(self, command, *args):
238
return self.client.juju(self, command, args)
239
240
def get_status(self):
241
return self.client.get_status(self)
242
243
def wait_for_started(self):
244
"""Wait until all unit/machine agents are 'started'."""
245
for ignored in until_timeout(1200):
246
try:
247
status = self.get_status()
248
except CannotConnectEnv:
249
print('Supressing "Unable to connect to environment"')
250
continue
251
states = status.check_agents_started(self.environment)
252
if states is None:
253
break
254
print(format_listing(states, 'started', self.environment))
255
sys.stdout.flush()
256
else:
257
raise Exception('Timed out waiting for agents to start in %s.' %
258
self.environment)
1573
259
return status
1574
260
1575
def wait_for_started(self, timeout=1200, start=None):
1576
"""Wait until all unit/machine agents are 'started'."""
1577
reporter = GroupReporter(sys.stdout, 'started')
1578
return self._wait_for_status(
1579
reporter, Status.check_agents_started, AgentsNotStarted,
1580
timeout=timeout, start=start)
1581
1582
def wait_for_subordinate_units(self, service, unit_prefix, timeout=1200,
1583
start=None):
1584
"""Wait until all service units have a started subordinate with
1585
unit_prefix."""
1586
def status_to_subordinate_states(status):
1587
service_unit_count = status.get_service_unit_count(service)
1588
subordinate_unit_count = 0
1589
unit_states = defaultdict(list)
1590
for name, unit in status.service_subordinate_units(service):
1591
if name.startswith(unit_prefix + '/'):
1592
subordinate_unit_count += 1
1593
unit_states[coalesce_agent_status(unit)].append(name)
1594
if (subordinate_unit_count == service_unit_count and
1595
set(unit_states.keys()).issubset(AGENTS_READY)):
1596
return None
1597
return unit_states
1598
reporter = GroupReporter(sys.stdout, 'started')
1599
self._wait_for_status(
1600
reporter, status_to_subordinate_states, AgentsNotStarted,
1601
timeout=timeout, start=start)
1602
1603
def wait_for_version(self, version, timeout=300, start=None):
1604
def status_to_version(status):
1605
versions = status.get_agent_versions()
261
def wait_for_version(self, version):
262
for ignored in until_timeout(300):
263
try:
264
versions = self.get_status().get_agent_versions()
265
except CannotConnectEnv:
266
print('Supressing "Unable to connect to environment"')
267
continue
1606
268
if versions.keys() == [version]:
1607
return None
1608
return versions
1609
reporter = GroupReporter(sys.stdout, version)
1610
self._wait_for_status(reporter, status_to_version, VersionsNotUpdated,
1611
timeout=timeout, start=start)
1612
1613
def list_models(self):
1614
"""List the models registered with the current controller."""
1615
self.controller_juju('list-models', ())
1616
1617
def get_models(self):
1618
"""return a models dict with a 'models': [] key-value pair.
1619
1620
The server has 120 seconds to respond because this method is called
1621
often when tearing down a controller-less deployment.
1622
"""
1623
output = self.get_juju_output(
1624
'list-models', '-c', self.env.controller.name, '--format', 'yaml',
1625
include_e=False, timeout=120)
1626
models = yaml_loads(output)
1627
return models
1628
1629
def _get_models(self):
1630
"""return a list of model dicts."""
1631
return self.get_models()['models']
1632
1633
def iter_model_clients(self):
1634
"""Iterate through all the models that share this model's controller.
1635
1636
Works only if JES is enabled.
1637
"""
1638
models = self._get_models()
1639
if not models:
1640
yield self
1641
for model in models:
1642
yield self._acquire_model_client(model['name'])
1643
1644
def get_controller_model_name(self):
1645
"""Return the name of the 'controller' model.
1646
1647
Return the name of the environment when an 'controller' model does
1648
not exist.
1649
"""
1650
return 'controller'
1651
1652
def _acquire_model_client(self, name):
1653
"""Get a client for a model with the supplied name.
1654
1655
If the name matches self, self is used. Otherwise, a clone is used.
1656
"""
1657
if name == self.env.environment:
1658
return self
269
break
270
print(format_listing(versions, version, self.environment))
271
sys.stdout.flush()
1659
272
else:
1660
env = self.env.clone(model_name=name)
1661
return self.clone(env=env)
1662
1663
def get_model_uuid(self):
1664
name = self.env.environment
1665
model = self._cmd_model(True, False)
1666
output_yaml = self.get_juju_output(
1667
'show-model', '--format', 'yaml', model, include_e=False)
1668
output = yaml.safe_load(output_yaml)
1669
return output[name]['model-uuid']
1670
1671
def get_controller_uuid(self):
1672
name = self.env.controller.name
1673
output_yaml = self.get_juju_output(
1674
'show-controller', '--format', 'yaml', include_e=False)
1675
output = yaml.safe_load(output_yaml)
1676
return output[name]['details']['uuid']
1677
1678
def get_controller_model_uuid(self):
1679
output_yaml = self.get_juju_output(
1680
'show-model', 'controller', '--format', 'yaml', include_e=False)
1681
output = yaml.safe_load(output_yaml)
1682
return output['controller']['model-uuid']
1683
1684
def get_controller_client(self):
1685
"""Return a client for the controller model. May return self.
1686
1687
This may be inaccurate for models created using add_model
1688
rather than bootstrap.
1689
"""
1690
return self._acquire_model_client(self.get_controller_model_name())
1691
1692
def list_controllers(self):
1693
"""List the controllers."""
1694
self.juju('list-controllers', (), include_e=False)
1695
1696
def get_controller_endpoint(self):
1697
"""Return the address of the controller leader."""
1698
controller = self.env.controller.name
1699
output = self.get_juju_output(
1700
'show-controller', controller, include_e=False)
1701
info = yaml_loads(output)
1702
endpoint = info[controller]['details']['api-endpoints'][0]
1703
address, port = split_address_port(endpoint)
1704
return address
1705
1706
def get_controller_members(self):
1707
"""Return a list of Machines that are members of the controller.
1708
1709
The first machine in the list is the leader. the remaining machines
1710
are followers in a HA relationship.
1711
"""
1712
members = []
1713
status = self.get_status()
1714
for machine_id, machine in status.iter_machines():
1715
if self.get_controller_member_status(machine):
1716
members.append(Machine(machine_id, machine))
1717
if len(members) <= 1:
1718
return members
1719
# Search for the leader and make it the first in the list.
1720
# If the endpoint address is not the same as the leader's dns_name,
1721
# the members are return in the order they were discovered.
1722
endpoint = self.get_controller_endpoint()
1723
log.debug('Controller endpoint is at {}'.format(endpoint))
1724
members.sort(key=lambda m: m.info.get('dns-name') != endpoint)
1725
return members
1726
1727
def get_controller_leader(self):
1728
"""Return the controller leader Machine."""
1729
controller_members = self.get_controller_members()
1730
return controller_members[0]
1731
1732
@staticmethod
1733
def get_controller_member_status(info_dict):
1734
"""Return the controller-member-status of the machine if it exists."""
1735
return info_dict.get('controller-member-status')
1736
1737
def wait_for_ha(self, timeout=1200):
1738
desired_state = 'has-vote'
1739
reporter = GroupReporter(sys.stdout, desired_state)
1740
try:
1741
with self.check_timeouts():
1742
with self.ignore_soft_deadline():
1743
for remaining in until_timeout(timeout):
1744
status = self.get_status(controller=True)
1745
status.check_agents_started()
1746
states = {}
1747
for machine, info in status.iter_machines():
1748
status = self.get_controller_member_status(info)
1749
if status is None:
1750
continue
1751
states.setdefault(status, []).append(machine)
1752
if states.keys() == [desired_state]:
1753
if len(states.get(desired_state, [])) >= 3:
1754
break
1755
reporter.update(states)
1756
else:
1757
raise Exception('Timed out waiting for voting to be'
1758
' enabled.')
1759
finally:
1760
reporter.finish()
1761
# XXX sinzui 2014-12-04: bug 1399277 happens because
1762
# juju claims HA is ready when the monogo replica sets
1763
# are not. Juju is not fully usable. The replica set
1764
# lag might be 5 minutes.
1765
self._backend.pause(300)
1766
1767
def wait_for_deploy_started(self, service_count=1, timeout=1200):
1768
"""Wait until service_count services are 'started'.
1769
1770
:param service_count: The number of services for which to wait.
1771
:param timeout: The number of seconds to wait.
1772
"""
1773
with self.check_timeouts():
1774
with self.ignore_soft_deadline():
1775
for remaining in until_timeout(timeout):
1776
status = self.get_status()
1777
if status.get_service_count() >= service_count:
1778
return
1779
else:
1780
raise Exception('Timed out waiting for services to start.')
1781
1782
def wait_for_workloads(self, timeout=600, start=None):
1783
"""Wait until all unit workloads are in a ready state."""
1784
def status_to_workloads(status):
1785
unit_states = defaultdict(list)
1786
for name, unit in status.iter_units():
1787
workload = unit.get('workload-status')
1788
if workload is not None:
1789
state = workload['current']
1790
else:
1791
state = 'unknown'
1792
unit_states[state].append(name)
1793
if set(('active', 'unknown')).issuperset(unit_states):
1794
return None
1795
unit_states.pop('unknown', None)
1796
return unit_states
1797
reporter = GroupReporter(sys.stdout, 'active')
1798
self._wait_for_status(reporter, status_to_workloads, WorkloadsNotReady,
1799
timeout=timeout, start=start)
1800
1801
def wait_for(self, thing, search_type, timeout=300):
1802
""" Wait for a something (thing) matching none/all/some machines.
1803
1804
Examples:
1805
wait_for('containers', 'all')
1806
This will wait for a container to appear on all machines.
1807
1808
wait_for('machines-not-0', 'none')
1809
This will wait for all machines other than 0 to be removed.
1810
1811
:param thing: string, either 'containers' or 'not-machine-0'
1812
:param search_type: string containing none, some or all
1813
:param timeout: number of seconds to wait for condition to be true.
1814
:return:
1815
"""
1816
try:
1817
for status in self.status_until(timeout):
1818
hit = False
1819
miss = False
1820
1821
for machine, details in status.status['machines'].iteritems():
1822
if thing == 'containers':
1823
if 'containers' in details:
1824
hit = True
1825
else:
1826
miss = True
1827
1828
elif thing == 'machines-not-0':
1829
if machine != '0':
1830
hit = True
1831
else:
1832
miss = True
1833
1834
else:
1835
raise ValueError("Unrecognised thing to wait for: %s",
1836
thing)
1837
1838
if search_type == 'none':
1839
if not hit:
1840
return
1841
elif search_type == 'some':
1842
if hit:
1843
return
1844
elif search_type == 'all':
1845
if not miss:
1846
return
1847
except Exception:
1848
raise Exception("Timed out waiting for %s" % thing)
273
raise Exception('Some versions did not update.')
1849
274
1850
275
def get_matching_agent_version(self, no_build=False):
1851
# strip the series and srch from the built version.
1852
version_parts = self.version.split('-')
1853
if len(version_parts) == 4:
1854
version_number = '-'.join(version_parts[0:2])
1855
else:
1856
version_number = version_parts[0]
1857
if not no_build and self.env.local:
276
version_number = self.client.version.split('-')[0]
277
if not no_build and self.local:
1858
278
version_number += '.1'
1859
279
return version_number
1860
280
1861
def upgrade_juju(self, force_version=True):
1862
args = ()
1863
if force_version:
1864
version = self.get_matching_agent_version(no_build=True)
1865
args += ('--agent-version', version)
1866
self._upgrade_juju(args)
1867
1868
def _upgrade_juju(self, args):
1869
self.juju('upgrade-juju', args)
1870
1871
def upgrade_mongo(self):
1872
self.juju('upgrade-mongo', ())
1873
1874
def backup(self):
1875
try:
1876
output = self.get_juju_output('create-backup')
1877
except subprocess.CalledProcessError as e:
1878
log.info(e.output)
1879
raise
1880
log.info(output)
1881
backup_file_pattern = re.compile('(juju-backup-[0-9-]+\.(t|tar.)gz)')
1882
match = backup_file_pattern.search(output)
1883
if match is None:
1884
raise Exception("The backup file was not found in output: %s" %
1885
output)
1886
backup_file_name = match.group(1)
1887
backup_file_path = os.path.abspath(backup_file_name)
1888
log.info("State-Server backup at %s", backup_file_path)
1889
return backup_file_path
1890
1891
def restore_backup(self, backup_file):
1892
self.juju(
1893
'restore-backup',
1894
('-b', '--constraints', 'mem=2G', '--file', backup_file))
1895
1896
def restore_backup_async(self, backup_file):
1897
return self.juju_async('restore-backup', ('-b', '--constraints',
1898
'mem=2G', '--file', backup_file))
1899
1900
def enable_ha(self):
1901
self.juju('enable-ha', ('-n', '3'))
1902
1903
def action_fetch(self, id, action=None, timeout="1m"):
1904
"""Fetches the results of the action with the given id.
1905
1906
Will wait for up to 1 minute for the action results.
1907
The action name here is just used for an more informational error in
1908
cases where it's available.
1909
Returns the yaml output of the fetched action.
1910
"""
1911
out = self.get_juju_output("show-action-output", id, "--wait", timeout)
1912
status = yaml_loads(out)["status"]
1913
if status != "completed":
1914
name = ""
1915
if action is not None:
1916
name = " " + action
1917
raise Exception(
1918
"timed out waiting for action%s to complete during fetch" %
1919
name)
1920
return out
1921
1922
def action_do(self, unit, action, *args):
1923
"""Performs the given action on the given unit.
1924
1925
Action params should be given as args in the form foo=bar.
1926
Returns the id of the queued action.
1927
"""
1928
args = (unit, action) + args
1929
1930
output = self.get_juju_output("run-action", *args)
1931
action_id_pattern = re.compile(
1932
'Action queued with id: ([a-f0-9\-]{36})')
1933
match = action_id_pattern.search(output)
1934
if match is None:
1935
raise Exception("Action id not found in output: %s" %
1936
output)
1937
return match.group(1)
1938
1939
def action_do_fetch(self, unit, action, timeout="1m", *args):
1940
"""Performs given action on given unit and waits for the results.
1941
1942
Action params should be given as args in the form foo=bar.
1943
Returns the yaml output of the action.
1944
"""
1945
id = self.action_do(unit, action, *args)
1946
return self.action_fetch(id, action, timeout)
1947
1948
def run(self, commands, applications):
1949
responses = self.get_juju_output(
1950
'run', '--format', 'json', '--application', ','.join(applications),
1951
*commands)
1952
return json.loads(responses)
1953
1954
def list_space(self):
1955
return yaml.safe_load(self.get_juju_output('list-space'))
1956
1957
def add_space(self, space):
1958
self.juju('add-space', (space),)
1959
1960
def add_subnet(self, subnet, space):
1961
self.juju('add-subnet', (subnet, space))
1962
1963
def is_juju1x(self):
1964
return self.version.startswith('1.')
1965
1966
def _get_register_command(self, output):
1967
"""Return register token from add-user output.
1968
1969
Return the register token supplied within the output from the add-user
1970
command.
1971
1972
"""
1973
for row in output.split('\n'):
1974
if 'juju register' in row:
1975
command_string = row.strip().lstrip()
1976
command_parts = command_string.split(' ')
1977
return command_parts[-1]
1978
raise AssertionError('Juju register command not found in output')
1979
1980
def add_user(self, username):
1981
"""Adds provided user and return register command arguments.
1982
1983
:return: Registration token provided by the add-user command.
1984
"""
1985
output = self.get_juju_output(
1986
'add-user', username, '-c', self.env.controller.name,
1987
include_e=False)
1988
return self._get_register_command(output)
1989
1990
def add_user_perms(self, username, models=None, permissions='login'):
1991
"""Adds provided user and return register command arguments.
1992
1993
:return: Registration token provided by the add-user command.
1994
"""
1995
output = self.add_user(username)
1996
self.grant(username, permissions, models)
1997
return output
1998
1999
def revoke(self, username, models=None, permissions='read'):
2000
if models is None:
2001
models = self.env.environment
2002
2003
args = (username, permissions, models)
2004
2005
self.controller_juju('revoke', args)
2006
2007
def add_storage(self, unit, storage_type, amount="1"):
2008
"""Add storage instances to service.
2009
2010
Only type 'disk' is able to add instances.
2011
"""
2012
self.juju('add-storage', (unit, storage_type + "=" + amount))
2013
2014
def list_storage(self):
2015
"""Return the storage list."""
2016
return self.get_juju_output('list-storage', '--format', 'json')
2017
2018
def list_storage_pool(self):
2019
"""Return the list of storage pool."""
2020
return self.get_juju_output('list-storage-pools', '--format', 'json')
2021
2022
def create_storage_pool(self, name, provider, size):
2023
"""Create storage pool."""
2024
self.juju('create-storage-pool',
2025
(name, provider,
2026
'size={}'.format(size)))
2027
2028
def disable_user(self, user_name):
2029
"""Disable an user"""
2030
self.controller_juju('disable-user', (user_name,))
2031
2032
def enable_user(self, user_name):
2033
"""Enable an user"""
2034
self.controller_juju('enable-user', (user_name,))
2035
2036
def logout(self):
2037
"""Logout an user"""
2038
self.controller_juju('logout', ())
2039
self.env.user_name = ''
2040
2041
def register_user(self, user, juju_home, controller_name=None):
2042
"""Register `user` for the `client` return the cloned client used."""
2043
username = user.name
2044
if controller_name is None:
2045
controller_name = '{}_controller'.format(username)
2046
2047
model = self.env.environment
2048
token = self.add_user_perms(username, models=model,
2049
permissions=user.permissions)
2050
user_client = self.create_cloned_environment(juju_home,
2051
controller_name,
2052
username)
2053
2054
try:
2055
child = user_client.expect('register', (token), include_e=False)
2056
child.expect('(?i)name')
2057
child.sendline(controller_name)
2058
child.expect('(?i)password')
2059
child.sendline(username + '_password')
2060
child.expect('(?i)password')
2061
child.sendline(username + '_password')
2062
child.expect(pexpect.EOF)
2063
if child.isalive():
2064
raise Exception(
2065
'Registering user failed: pexpect session still alive')
2066
except pexpect.TIMEOUT:
2067
raise Exception(
2068
'Registering user failed: pexpect session timed out')
2069
user_client.env.user_name = username
2070
return user_client
2071
2072
def remove_user(self, username):
2073
self.juju('remove-user', (username, '-y'), include_e=False)
2074
2075
def create_cloned_environment(
2076
self, cloned_juju_home, controller_name, user_name=None):
2077
"""Create a cloned environment.
2078
2079
If `user_name` is passed ensures that the cloned environment is updated
2080
to match.
2081
2082
"""
2083
user_client = self.clone(env=self.env.clone())
2084
user_client.env.juju_home = cloned_juju_home
2085
if user_name is not None and user_name != self.env.user_name:
2086
user_client.env.user_name = user_name
2087
user_client.env.environment = qualified_model_name(
2088
user_client.env.environment, self.env.user_name)
2089
# New user names the controller.
2090
user_client.env.controller = Controller(controller_name)
2091
return user_client
2092
2093
def grant(self, user_name, permission, model=None):
2094
"""Grant the user with model or controller permission."""
2095
if permission in self.controller_permissions:
2096
self.juju(
2097
'grant',
2098
(user_name, permission, '-c', self.env.controller.name),
2099
include_e=False)
2100
elif permission in self.model_permissions:
2101
if model is None:
2102
model = self.model_name
2103
self.juju(
2104
'grant',
2105
(user_name, permission, model, '-c', self.env.controller.name),
2106
include_e=False)
2107
else:
2108
raise ValueError('Unknown permission {}'.format(permission))
2109
2110
def list_clouds(self, format='json'):
2111
"""List all the available clouds."""
2112
return self.get_juju_output('list-clouds', '--format',
2113
format, include_e=False)
2114
2115
def show_controller(self, format='json'):
2116
"""Show controller's status."""
2117
return self.get_juju_output('show-controller', '--format',
2118
format, include_e=False)
2119
2120
def ssh_keys(self, full=False):
2121
"""Give the ssh keys registered for the current model."""
2122
args = []
2123
if full:
2124
args.append('--full')
2125
return self.get_juju_output('ssh-keys', *args)
2126
2127
def add_ssh_key(self, *keys):
2128
"""Add one or more ssh keys to the current model."""
2129
return self.get_juju_output('add-ssh-key', *keys, merge_stderr=True)
2130
2131
def remove_ssh_key(self, *keys):
2132
"""Remove one or more ssh keys from the current model."""
2133
return self.get_juju_output('remove-ssh-key', *keys, merge_stderr=True)
2134
2135
def import_ssh_key(self, *keys):
2136
"""Import ssh keys from one or more identities to the current model."""
2137
return self.get_juju_output('import-ssh-key', *keys, merge_stderr=True)
2138
2139
def list_disabled_commands(self):
2140
"""List all the commands disabled on the model."""
2141
raw = self.get_juju_output('list-disabled-commands',
2142
'--format', 'yaml')
2143
return yaml.safe_load(raw)
2144
2145
def disable_command(self, args):
2146
"""Disable a command set."""
2147
return self.juju('disable-command', args)
2148
2149
def enable_command(self, args):
2150
"""Enable a command set."""
2151
return self.juju('enable-command', args)
2152
2153
def sync_tools(self, local_dir=None):
2154
"""Copy tools into a local directory or model."""
2155
if local_dir is None:
2156
return self.juju('sync-tools', ())
2157
else:
2158
return self.juju('sync-tools', ('--local-dir', local_dir),
2159
include_e=False)
2160
2161
2162
class EnvJujuClient2B9(EnvJujuClient):
2163
2164
def update_user_name(self):
2165
return
2166
2167
def create_cloned_environment(
2168
self, cloned_juju_home, controller_name, user_name=None):
2169
"""Create a cloned environment.
2170
2171
`user_name` is unused in this version of beta.
2172
"""
2173
user_client = self.clone(env=self.env.clone())
2174
user_client.env.juju_home = cloned_juju_home
2175
# New user names the controller.
2176
user_client.env.controller = Controller(controller_name)
2177
return user_client
2178
2179
def get_model_uuid(self):
2180
name = self.env.environment
2181
output_yaml = self.get_juju_output('show-model', '--format', 'yaml')
2182
output = yaml.safe_load(output_yaml)
2183
return output[name]['model-uuid']
2184
2185
def add_user_perms(self, username, models=None, permissions='read'):
2186
"""Adds provided user and return register command arguments.
2187
2188
:return: Registration token provided by the add-user command.
2189
2190
"""
2191
if models is None:
2192
models = self.env.environment
2193
2194
args = (username, '--models', models, '--acl', permissions,
2195
'-c', self.env.controller.name)
2196
2197
output = self.get_juju_output('add-user', *args, include_e=False)
2198
return self._get_register_command(output)
2199
2200
def grant(self, user_name, permission, model=None):
2201
"""Grant the user with a model."""
2202
if model is None:
2203
model = self.model_name
2204
self.juju('grant', (user_name, model, '--acl', permission),
2205
include_e=False)
2206
2207
def revoke(self, username, models=None, permissions='read'):
2208
if models is None:
2209
models = self.env.environment
2210
2211
args = (username, models, '--acl', permissions)
2212
2213
self.controller_juju('revoke', args)
2214
2215
def set_config(self, service, options):
2216
option_strings = self._dict_as_option_strings(options)
2217
self.juju('set-config', (service,) + option_strings)
2218
2219
def get_config(self, service):
2220
return yaml_loads(self.get_juju_output('get-config', service))
2221
2222
def get_model_config(self):
2223
"""Return the value of the environment's configured option."""
2224
return yaml.safe_load(
2225
self.get_juju_output('get-model-config', '--format', 'yaml'))
2226
2227
def get_env_option(self, option):
2228
"""Return the value of the environment's configured option."""
2229
return self.get_juju_output('get-model-config', option)
2230
2231
def set_env_option(self, option, value):
2232
"""Set the value of the option in the environment."""
2233
option_value = "%s=%s" % (option, value)
2234
return self.juju('set-model-config', (option_value,))
2235
2236
def unset_env_option(self, option):
2237
"""Unset the value of the option in the environment."""
2238
return self.juju('unset-model-config', (option,))
2239
2240
def list_disabled_commands(self):
2241
"""List all the commands disabled on the model."""
2242
raw = self.get_juju_output('block list', '--format', 'yaml')
2243
return yaml.safe_load(raw)
2244
2245
def disable_command(self, args):
2246
"""Disable a command set."""
2247
return self.juju('block', args)
2248
2249
def enable_command(self, args):
2250
"""Enable a command set."""
2251
return self.juju('unblock', args)
2252
2253
2254
class EnvJujuClient2B8(EnvJujuClient2B9):
2255
2256
status_class = ServiceStatus
2257
2258
def remove_service(self, service):
2259
self.juju('remove-service', (service,))
2260
2261
def run(self, commands, applications):
2262
responses = self.get_juju_output(
2263
'run', '--format', 'json', '--service', ','.join(applications),
2264
*commands)
2265
return json.loads(responses)
2266
2267
def deployer(self, bundle_template, name=None, deploy_delay=10,
2268
timeout=3600):
2269
"""Deploy a bundle using deployer."""
2270
bundle = self.format_bundle(bundle_template)
2271
args = (
2272
'--debug',
2273
'--deploy-delay', str(deploy_delay),
2274
'--timeout', str(timeout),
2275
'--config', bundle,
2276
)
2277
if name:
2278
args += (name,)
2279
e_arg = ('-e', 'local.{}:{}'.format(
2280
self.env.controller.name, self.env.environment))
2281
args = e_arg + args
2282
self.juju('deployer', args, self.env.needs_sudo(), include_e=False)
2283
2284
2285
class EnvJujuClient2B7(EnvJujuClient2B8):
2286
2287
def get_controller_model_name(self):
2288
"""Return the name of the 'controller' model.
2289
2290
Return the name of the environment when an 'controller' model does
2291
not exist.
2292
"""
2293
return 'admin'
2294
2295
2296
class EnvJujuClient2B3(EnvJujuClient2B7):
2297
2298
def _add_model(self, model_name, config_file):
2299
self.controller_juju('create-model', (
2300
model_name, '--config', config_file))
2301
2302
2303
class EnvJujuClient2B2(EnvJujuClient2B3):
2304
2305
def get_bootstrap_args(
2306
self, upload_tools, config_filename, bootstrap_series=None,
2307
credential=None, auto_upgrade=False, metadata_source=None,
2308
to=None, agent_version=None):
2309
"""Return the bootstrap arguments for the substrate."""
2310
err_fmt = 'EnvJujuClient2B2 does not support bootstrap argument {}'
2311
if auto_upgrade:
2312
raise ValueError(err_fmt.format('auto_upgrade'))
2313
if metadata_source is not None:
2314
raise ValueError(err_fmt.format('metadata_source'))
2315
if to is not None:
2316
raise ValueError(err_fmt.format('to'))
2317
if agent_version is not None:
2318
raise ValueError(err_fmt.format('agent_version'))
2319
if self.env.joyent:
2320
# Only accept kvm packages by requiring >1 cpu core, see lp:1446264
2321
constraints = 'mem=2G cpu-cores=1'
2322
else:
2323
constraints = 'mem=2G'
2324
cloud_region = self.get_cloud_region(self.env.get_cloud(),
2325
self.env.get_region())
2326
args = ['--constraints', constraints, self.env.environment,
2327
cloud_region, '--config', config_filename]
2328
if upload_tools:
2329
args.insert(0, '--upload-tools')
2330
else:
2331
args.extend(['--agent-version', self.get_matching_agent_version()])
2332
2333
if bootstrap_series is not None:
2334
args.extend(['--bootstrap-series', bootstrap_series])
2335
2336
if credential is not None:
2337
args.extend(['--credential', credential])
2338
2339
return tuple(args)
2340
2341
def get_controller_client(self):
2342
"""Return a client for the controller model. May return self."""
2343
return self
2344
2345
def get_controller_model_name(self):
2346
"""Return the name of the 'controller' model.
2347
2348
Return the name of the environment when an 'controller' model does
2349
not exist.
2350
"""
2351
models = self.get_models()
2352
# The dict can be empty because 1.x does not support the models.
2353
# This is an ambiguous case for the jes feature flag which supports
2354
# multiple models, but none is named 'admin' by default. Since the
2355
# jes case also uses '-e' for models, the env is the controller model.
2356
for model in models.get('models', []):
2357
if 'admin' in model['name']:
2358
return 'admin'
2359
return self.env.environment
2360
2361
2362
class EnvJujuClient2A2(EnvJujuClient2B2):
2363
"""Drives Juju 2.0-alpha2 clients."""
2364
2365
default_backend = Juju2A2Backend
2366
2367
config_class = SimpleEnvironment
2368
2369
@classmethod
2370
def _get_env(cls, env):
2371
if isinstance(env, JujuData):
2372
raise IncompatibleConfigClass(
2373
'JujuData cannot be used with {}'.format(cls.__name__))
2374
return env
2375
2376
def bootstrap(self, upload_tools=False, bootstrap_series=None):
2377
"""Bootstrap a controller."""
2378
self._check_bootstrap()
2379
args = self.get_bootstrap_args(upload_tools, bootstrap_series)
2380
self.juju('bootstrap', args, self.env.needs_sudo())
2381
2382
@contextmanager
2383
def bootstrap_async(self, upload_tools=False):
2384
self._check_bootstrap()
2385
args = self.get_bootstrap_args(upload_tools)
2386
with self.juju_async('bootstrap', args):
2387
yield
2388
log.info('Waiting for bootstrap of {}.'.format(
2389
self.env.environment))
2390
2391
def get_bootstrap_args(self, upload_tools, bootstrap_series=None,
2392
credential=None):
2393
"""Return the bootstrap arguments for the substrate."""
2394
if credential is not None:
2395
raise ValueError(
2396
'--credential is not supported by this juju version.')
2397
constraints = self._get_substrate_constraints()
2398
args = ('--constraints', constraints,
2399
'--agent-version', self.get_matching_agent_version())
2400
if upload_tools:
2401
args = ('--upload-tools',) + args
2402
if bootstrap_series is not None:
2403
args = args + ('--bootstrap-series', bootstrap_series)
2404
return args
2405
2406
def deploy(self, charm, repository=None, to=None, series=None,
2407
service=None, force=False, storage=None, constraints=None):
2408
args = [charm]
2409
if repository is not None:
2410
args.extend(['--repository', repository])
2411
if to is not None:
2412
args.extend(['--to', to])
2413
if service is not None:
2414
args.extend([service])
2415
if storage is not None:
2416
args.extend(['--storage', storage])
2417
if constraints is not None:
2418
args.extend(['--constraints', constraints])
2419
return self.juju('deploy', tuple(args))
2420
2421
2422
class EnvJujuClient2A1(EnvJujuClient2A2):
2423
"""Drives Juju 2.0-alpha1 clients."""
2424
2425
_show_status = 'status'
2426
2427
default_backend = Juju1XBackend
2428
2429
def get_cache_path(self):
2430
return get_cache_path(self.env.juju_home, models=False)
2431
2432
def remove_service(self, service):
2433
self.juju('destroy-service', (service,))
2434
2435
def backup(self):
2436
environ = self._shell_environ()
2437
# juju-backup does not support the -e flag.
2438
environ['JUJU_ENV'] = self.env.environment
2439
try:
2440
# Mutate os.environ instead of supplying env parameter so Windows
2441
# can search env['PATH']
2442
with scoped_environ(environ):
2443
args = ['juju', 'backup']
2444
log.info(' '.join(args))
2445
output = subprocess.check_output(args)
2446
except subprocess.CalledProcessError as e:
2447
log.info(e.output)
2448
raise
2449
log.info(output)
2450
backup_file_pattern = re.compile('(juju-backup-[0-9-]+\.(t|tar.)gz)')
2451
match = backup_file_pattern.search(output)
2452
if match is None:
2453
raise Exception("The backup file was not found in output: %s" %
2454
output)
2455
backup_file_name = match.group(1)
2456
backup_file_path = os.path.abspath(backup_file_name)
2457
log.info("State-Server backup at %s", backup_file_path)
2458
return backup_file_path
2459
2460
def restore_backup(self, backup_file):
2461
return self.get_juju_output('restore', '--constraints', 'mem=2G',
2462
backup_file)
2463
2464
def restore_backup_async(self, backup_file):
2465
return self.juju_async('restore', ('--constraints', 'mem=2G',
2466
backup_file))
2467
2468
def enable_ha(self):
2469
self.juju('ensure-availability', ('-n', '3'))
2470
2471
def list_models(self):
2472
"""List the models registered with the current controller."""
2473
log.info('The model is environment {}'.format(self.env.environment))
2474
2475
def list_clouds(self, format='json'):
2476
"""List all the available clouds."""
2477
return {}
2478
2479
def show_controller(self, format='json'):
2480
"""Show controller's status."""
2481
return {}
2482
2483
def get_models(self):
2484
"""return a models dict with a 'models': [] key-value pair."""
2485
return {}
2486
2487
def _get_models(self):
2488
"""return a list of model dicts."""
2489
# In 2.0-alpha1, 'list-models' produced a yaml list rather than a
2490
# dict, but the command and parsing are the same.
2491
return super(EnvJujuClient2A1, self).get_models()
2492
2493
def list_controllers(self):
2494
"""List the controllers."""
2495
log.info(
2496
'The controller is environment {}'.format(self.env.environment))
2497
2498
@staticmethod
2499
def get_controller_member_status(info_dict):
2500
return info_dict.get('state-server-member-status')
2501
2502
def action_fetch(self, id, action=None, timeout="1m"):
2503
"""Fetches the results of the action with the given id.
2504
2505
Will wait for up to 1 minute for the action results.
2506
The action name here is just used for an more informational error in
2507
cases where it's available.
2508
Returns the yaml output of the fetched action.
2509
"""
2510
# the command has to be "action fetch" so that the -e <env> args are
2511
# placed after "fetch", since that's where action requires them to be.
2512
out = self.get_juju_output("action fetch", id, "--wait", timeout)
2513
status = yaml_loads(out)["status"]
2514
if status != "completed":
2515
name = ""
2516
if action is not None:
2517
name = " " + action
2518
raise Exception(
2519
"timed out waiting for action%s to complete during fetch" %
2520
name)
2521
return out
2522
2523
def action_do(self, unit, action, *args):
2524
"""Performs the given action on the given unit.
2525
2526
Action params should be given as args in the form foo=bar.
2527
Returns the id of the queued action.
2528
"""
2529
args = (unit, action) + args
2530
2531
# the command has to be "action do" so that the -e <env> args are
2532
# placed after "do", since that's where action requires them to be.
2533
output = self.get_juju_output("action do", *args)
2534
action_id_pattern = re.compile(
2535
'Action queued with id: ([a-f0-9\-]{36})')
2536
match = action_id_pattern.search(output)
2537
if match is None:
2538
raise Exception("Action id not found in output: %s" %
2539
output)
2540
return match.group(1)
2541
2542
def list_space(self):
2543
return yaml.safe_load(self.get_juju_output('space list'))
2544
2545
def add_space(self, space):
2546
self.juju('space create', (space),)
2547
2548
def add_subnet(self, subnet, space):
2549
self.juju('subnet add', (subnet, space))
2550
2551
def set_model_constraints(self, constraints):
2552
constraint_strings = self._dict_as_option_strings(constraints)
2553
return self.juju('set-constraints', constraint_strings)
2554
2555
def set_config(self, service, options):
2556
option_strings = ['{}={}'.format(*item) for item in options.items()]
2557
self.juju('set', (service,) + tuple(option_strings))
2558
2559
def get_config(self, service):
2560
return yaml_loads(self.get_juju_output('get', service))
2561
2562
def get_model_config(self):
2563
"""Return the value of the environment's configured option."""
2564
return yaml.safe_load(self.get_juju_output('get-env'))
2565
2566
def get_env_option(self, option):
2567
"""Return the value of the environment's configured option."""
2568
return self.get_juju_output('get-env', option)
2569
2570
def set_env_option(self, option, value):
2571
"""Set the value of the option in the environment."""
2572
option_value = "%s=%s" % (option, value)
2573
return self.juju('set-env', (option_value,))
2574
2575
2576
class EnvJujuClient1X(EnvJujuClient2A1):
2577
"""Base for all 1.x client drivers."""
2578
2579
# The environments.yaml options that are replaced by bootstrap options.
2580
# For Juju 1.x, no bootstrap options are used.
2581
bootstrap_replaces = frozenset()
2582
2583
destroy_model_command = 'destroy-environment'
2584
2585
supported_container_types = frozenset([KVM_MACHINE, LXC_MACHINE])
2586
2587
agent_metadata_url = 'tools-metadata-url'
2588
2589
def _cmd_model(self, include_e, controller):
2590
if controller:
2591
return self.get_controller_model_name()
2592
elif self.env is None or not include_e:
2593
return None
2594
else:
2595
return unqualified_model_name(self.model_name)
2596
2597
def get_bootstrap_args(self, upload_tools, bootstrap_series=None,
2598
credential=None):
2599
"""Return the bootstrap arguments for the substrate."""
2600
if credential is not None:
2601
raise ValueError(
2602
'--credential is not supported by this juju version.')
2603
constraints = self._get_substrate_constraints()
2604
args = ('--constraints', constraints)
2605
if upload_tools:
2606
args = ('--upload-tools',) + args
2607
if bootstrap_series is not None:
2608
env_val = self.env.config.get('default-series')
2609
if bootstrap_series != env_val:
2610
raise BootstrapMismatch(
2611
'bootstrap-series', bootstrap_series, 'default-series',
2612
env_val)
2613
return args
2614
2615
def get_jes_command(self):
2616
"""Return the JES command to destroy a controller.
2617
2618
Juju 2.x has 'kill-controller'.
2619
Some intermediate versions had 'controller kill'.
2620
Juju 1.25 has 'system kill' when the jes feature flag is set.
2621
2622
:raises: JESNotSupported when the version of Juju does not expose
2623
a JES command.
2624
:return: The JES command.
2625
"""
2626
commands = self.get_juju_output('help', 'commands', include_e=False)
2627
for line in commands.splitlines():
2628
for cmd in _jes_cmds.keys():
2629
if line.startswith(cmd):
2630
return cmd
2631
raise JESNotSupported()
2632
2633
def upgrade_juju(self, force_version=True):
2634
args = ()
2635
if force_version:
2636
version = self.get_matching_agent_version(no_build=True)
2637
args += ('--version', version)
2638
if self.env.local:
2639
args += ('--upload-tools',)
2640
self._upgrade_juju(args)
2641
2642
def make_model_config(self):
2643
config_dict = make_safe_config(self)
2644
# Strip unneeded variables.
2645
return config_dict
2646
2647
def _add_model(self, model_name, config_file):
2648
seen_cmd = self.get_jes_command()
2649
if seen_cmd == SYSTEM:
2650
controller_option = ('-s', self.env.environment)
2651
else:
2652
controller_option = ('-c', self.env.environment)
2653
self.juju(_jes_cmds[seen_cmd]['create'], controller_option + (
2654
model_name, '--config', config_file), include_e=False)
2655
2656
def destroy_model(self):
2657
"""With JES enabled, destroy-environment destroys the model."""
2658
self.destroy_environment(force=False)
2659
2660
def destroy_environment(self, force=True, delete_jenv=False):
2661
if force:
2662
force_arg = ('--force',)
2663
else:
2664
force_arg = ()
2665
exit_status = self.juju(
2666
'destroy-environment',
2667
(self.env.environment,) + force_arg + ('-y',),
2668
self.env.needs_sudo(), check=False, include_e=False,
2669
timeout=get_teardown_timeout(self))
2670
if delete_jenv:
2671
jenv_path = get_jenv_path(self.env.juju_home, self.env.environment)
2672
ensure_deleted(jenv_path)
2673
return exit_status
2674
2675
def _get_models(self):
2676
"""return a list of model dicts."""
2677
try:
2678
return yaml.safe_load(self.get_juju_output(
2679
'environments', '-s', self.env.environment, '--format', 'yaml',
2680
include_e=False))
2681
except subprocess.CalledProcessError:
2682
# This *private* method attempts to use a 1.25 JES feature.
2683
# The JES design is dead. The private method is not used to
2684
# directly test juju cli; the failure is not a contract violation.
2685
log.info('Call to JES juju environments failed, falling back.')
2686
return []
2687
2688
def deploy_bundle(self, bundle, timeout=_DEFAULT_BUNDLE_TIMEOUT):
2689
"""Deploy bundle using deployer for Juju 1.X version."""
2690
self.deployer(bundle, timeout=timeout)
2691
2692
def deployer(self, bundle_template, name=None, deploy_delay=10,
2693
timeout=3600):
2694
"""Deploy a bundle using deployer."""
2695
bundle = self.format_bundle(bundle_template)
2696
args = (
2697
'--debug',
2698
'--deploy-delay', str(deploy_delay),
2699
'--timeout', str(timeout),
2700
'--config', bundle,
2701
)
2702
if name:
2703
args += (name,)
2704
self.juju('deployer', args, self.env.needs_sudo())
2705
2706
def upgrade_charm(self, service, charm_path=None):
2707
args = (service,)
2708
if charm_path is not None:
2709
repository = os.path.dirname(os.path.dirname(charm_path))
2710
args = args + ('--repository', repository)
2711
self.juju('upgrade-charm', args)
2712
2713
def get_controller_endpoint(self):
2714
"""Return the address of the state-server leader."""
2715
endpoint = self.get_juju_output('api-endpoints')
2716
address, port = split_address_port(endpoint)
2717
return address
2718
2719
def upgrade_mongo(self):
2720
raise UpgradeMongoNotSupported()
2721
2722
def add_storage(self, unit, storage_type, amount="1"):
2723
"""Add storage instances to service.
2724
2725
Only type 'disk' is able to add instances.
2726
"""
2727
self.juju('storage add', (unit, storage_type + "=" + amount))
2728
2729
def list_storage(self):
2730
"""Return the storage list."""
2731
return self.get_juju_output('storage list', '--format', 'json')
2732
2733
def list_storage_pool(self):
2734
"""Return the list of storage pool."""
2735
return self.get_juju_output('storage pool list', '--format', 'json')
2736
2737
def create_storage_pool(self, name, provider, size):
2738
"""Create storage pool."""
2739
self.juju('storage pool create',
2740
(name, provider,
2741
'size={}'.format(size)))
2742
2743
def ssh_keys(self, full=False):
2744
"""Give the ssh keys registered for the current model."""
2745
args = []
2746
if full:
2747
args.append('--full')
2748
return self.get_juju_output('authorized-keys list', *args)
2749
2750
def add_ssh_key(self, *keys):
2751
"""Add one or more ssh keys to the current model."""
2752
return self.get_juju_output('authorized-keys add', *keys,
2753
merge_stderr=True)
2754
2755
def remove_ssh_key(self, *keys):
2756
"""Remove one or more ssh keys from the current model."""
2757
return self.get_juju_output('authorized-keys delete', *keys,
2758
merge_stderr=True)
2759
2760
def import_ssh_key(self, *keys):
2761
"""Import ssh keys from one or more identities to the current model."""
2762
return self.get_juju_output('authorized-keys import', *keys,
2763
merge_stderr=True)
2764
2765
2766
class EnvJujuClient22(EnvJujuClient1X):
2767
2768
used_feature_flags = frozenset(['actions'])
2769
2770
def __init__(self, *args, **kwargs):
2771
super(EnvJujuClient22, self).__init__(*args, **kwargs)
2772
self.feature_flags.add('actions')
2773
2774
2775
class EnvJujuClient26(EnvJujuClient1X):
2776
"""Drives Juju 2.6-series clients."""
2777
2778
used_feature_flags = frozenset(['address-allocation', 'cloudsigma', 'jes'])
2779
2780
def __init__(self, *args, **kwargs):
2781
super(EnvJujuClient26, self).__init__(*args, **kwargs)
2782
if self.env is None or self.env.config is None:
2783
return
2784
if self.env.config.get('type') == 'cloudsigma':
2785
self.feature_flags.add('cloudsigma')
2786
2787
def enable_jes(self):
2788
"""Enable JES if JES is optional.
2789
2790
:raises: JESByDefault when JES is always enabled; Juju has the
2791
'destroy-controller' command.
2792
:raises: JESNotSupported when JES is not supported; Juju does not have
2793
the 'system kill' command when the JES feature flag is set.
2794
"""
2795
2796
if 'jes' in self.feature_flags:
2797
return
2798
if self.is_jes_enabled():
2799
raise JESByDefault()
2800
self.feature_flags.add('jes')
2801
if not self.is_jes_enabled():
2802
self.feature_flags.remove('jes')
2803
raise JESNotSupported()
2804
2805
def disable_jes(self):
2806
if 'jes' in self.feature_flags:
2807
self.feature_flags.remove('jes')
2808
2809
def enable_container_address_allocation(self):
2810
self.feature_flags.add('address-allocation')
2811
2812
2813
class EnvJujuClient25(EnvJujuClient26):
2814
"""Drives Juju 2.5-series clients."""
2815
2816
2817
class EnvJujuClient24(EnvJujuClient25):
2818
"""Similar to EnvJujuClient25, but lacking JES support."""
2819
2820
used_feature_flags = frozenset(['cloudsigma'])
2821
2822
def enable_jes(self):
2823
raise JESNotSupported()
2824
2825
def add_ssh_machines(self, machines):
2826
for machine in machines:
2827
self.juju('add-machine', ('ssh:' + machine,))
2828
2829
2830
def get_local_root(juju_home, env):
2831
return os.path.join(juju_home, env.environment)
2832
2833
2834
def bootstrap_from_env(juju_home, client):
2835
with temp_bootstrap_env(juju_home, client):
2836
client.bootstrap()
2837
2838
2839
def quickstart_from_env(juju_home, client, bundle):
2840
with temp_bootstrap_env(juju_home, client):
2841
client.quickstart(bundle)
2842
2843
2844
@contextmanager
2845
def maybe_jes(client, jes_enabled, try_jes):
2846
"""If JES is desired and not enabled, try to enable it for this context.
2847
2848
JES will be in its previous state after exiting this context.
2849
If jes_enabled is True or try_jes is False, the context is a no-op.
2850
If enable_jes() raises JESNotSupported, JES will not be enabled in the
2851
context.
2852
2853
The with value is True if JES is enabled in the context.
2854
"""
2855
2856
class JESUnwanted(Exception):
2857
"""Non-error. Used to avoid enabling JES if not wanted."""
2858
2859
try:
2860
if not try_jes or jes_enabled:
2861
raise JESUnwanted
2862
client.enable_jes()
2863
except (JESNotSupported, JESUnwanted):
2864
yield jes_enabled
2865
return
2866
else:
2867
try:
2868
yield True
2869
finally:
2870
client.disable_jes()
2871
2872
2873
def tear_down(client, jes_enabled, try_jes=False):
2874
"""Tear down a JES or non-JES environment.
2875
2876
JES environments are torn down via 'controller kill' or 'system kill',
2877
and non-JES environments are torn down via 'destroy-environment --force.'
2878
"""
2879
with maybe_jes(client, jes_enabled, try_jes) as jes_enabled:
2880
if jes_enabled:
2881
client.kill_controller()
2882
else:
2883
if client.destroy_environment(force=False) != 0:
2884
client.destroy_environment(force=True)
2885
2886
2887
def uniquify_local(env):
2888
"""Ensure that local environments have unique port settings.
2889
2890
This allows local environments to be duplicated despite
2891
https://bugs.launchpad.net/bugs/1382131
2892
"""
2893
if not env.local:
2894
return
2895
port_defaults = {
2896
'api-port': 17070,
2897
'state-port': 37017,
2898
'storage-port': 8040,
2899
'syslog-port': 6514,
2900
}
2901
for key, default in port_defaults.items():
2902
env.config[key] = env.config.get(key, default) + 1
2903
2904
2905
def dump_environments_yaml(juju_home, config):
2906
"""Dump yaml data to the environment file.
2907
2908
:param juju_home: Path to the JUJU_HOME directory.
2909
:param config: Dictionary repersenting yaml data to dump."""
2910
environments_path = get_environments_path(juju_home)
2911
with open(environments_path, 'w') as config_file:
2912
yaml.safe_dump(config, config_file)
2913
2914
2915
@contextmanager
2916
def _temp_env(new_config, parent=None, set_home=True):
2917
"""Use the supplied config as juju environment.
2918
2919
This is not a fully-formed version for bootstrapping. See
2920
temp_bootstrap_env.
2921
"""
2922
with temp_dir(parent) as temp_juju_home:
2923
dump_environments_yaml(temp_juju_home, new_config)
2924
if set_home:
2925
context = scoped_environ()
2926
else:
2927
context = nested()
2928
with context:
2929
if set_home:
2930
os.environ['JUJU_HOME'] = temp_juju_home
2931
os.environ['JUJU_DATA'] = temp_juju_home
2932
yield temp_juju_home
2933
2934
2935
def jes_home_path(juju_home, dir_name):
2936
return os.path.join(juju_home, 'jes-homes', dir_name)
2937
2938
2939
def get_cache_path(juju_home, models=False):
2940
if models:
2941
root = os.path.join(juju_home, 'models')
2942
else:
2943
root = os.path.join(juju_home, 'environments')
2944
return os.path.join(root, 'cache.yaml')
2945
2946
2947
def make_safe_config(client):
2948
config = dict(client.env.config)
2949
if 'agent-version' in client.bootstrap_replaces:
2950
config.pop('agent-version', None)
2951
else:
2952
config['agent-version'] = client.get_matching_agent_version()
2953
# AFAICT, we *always* want to set test-mode to True. If we ever find a
2954
# use-case where we don't, we can make this optional.
2955
config['test-mode'] = True
2956
# Explicitly set 'name', which Juju implicitly sets to env.environment to
2957
# ensure MAASAccount knows what the name will be.
2958
config['name'] = unqualified_model_name(client.env.environment)
2959
if config['type'] == 'local':
2960
config.setdefault('root-dir', get_local_root(client.env.juju_home,
2961
client.env))
2962
# MongoDB requires a lot of free disk space, and the only
2963
# visible error message is from "juju bootstrap":
2964
# "cannot initiate replication set" if disk space is low.
2965
# What "low" exactly means, is unclear, but 8GB should be
2966
# enough.
2967
ensure_dir(config['root-dir'])
2968
check_free_disk_space(config['root-dir'], 8000000, "MongoDB files")
2969
if client.env.kvm:
2970
check_free_disk_space(
2971
"/var/lib/uvtool/libvirt/images", 2000000,
2972
"KVM disk files")
2973
else:
2974
check_free_disk_space(
2975
"/var/lib/lxc", 2000000, "LXC containers")
2976
return config
2977
2978
2979
@contextmanager
2980
def temp_bootstrap_env(juju_home, client, set_home=True, permanent=False):
2981
"""Create a temporary environment for bootstrapping.
2982
2983
This involves creating a temporary juju home directory and returning its
2984
location.
2985
2986
:param set_home: Set JUJU_HOME to match the temporary home in this
2987
context. If False, juju_home should be supplied to bootstrap.
2988
"""
2989
new_config = {
2990
'environments': {client.env.environment: make_safe_config(client)}}
2991
# Always bootstrap a matching environment.
2992
jenv_path = get_jenv_path(juju_home, client.env.environment)
2993
if permanent:
2994
context = client.env.make_jes_home(
2995
juju_home, client.env.environment, new_config)
2996
else:
2997
context = _temp_env(new_config, juju_home, set_home)
2998
with context as temp_juju_home:
2999
if os.path.lexists(jenv_path):
3000
raise Exception('%s already exists!' % jenv_path)
3001
new_jenv_path = get_jenv_path(temp_juju_home, client.env.environment)
3002
# Create a symlink to allow access while bootstrapping, and to reduce
3003
# races. Can't use a hard link because jenv doesn't exist until
3004
# partway through bootstrap.
3005
ensure_dir(os.path.join(juju_home, 'environments'))
3006
# Skip creating symlink where not supported (i.e. Windows).
3007
if not permanent and getattr(os, 'symlink', None) is not None:
3008
os.symlink(new_jenv_path, jenv_path)
3009
old_juju_home = client.env.juju_home
3010
client.env.juju_home = temp_juju_home
3011
try:
3012
yield temp_juju_home
3013
finally:
3014
if not permanent:
3015
# replace symlink with file before deleting temp home.
3016
try:
3017
os.rename(new_jenv_path, jenv_path)
3018
except OSError as e:
3019
if e.errno != errno.ENOENT:
3020
raise
3021
# Remove dangling symlink
3022
try:
3023
os.unlink(jenv_path)
3024
except OSError as e:
3025
if e.errno != errno.ENOENT:
3026
raise
3027
client.env.juju_home = old_juju_home
3028
3029
3030
def get_machine_dns_name(client, machine, timeout=600):
3031
"""Wait for dns-name on a juju machine."""
3032
for status in client.status_until(timeout=timeout):
3033
try:
3034
return _dns_name_for_machine(status, machine)
3035
except KeyError:
3036
log.debug("No dns-name yet for machine %s", machine)
3037
3038
3039
def _dns_name_for_machine(status, machine):
3040
host = status.status['machines'][machine]['dns-name']
3041
if is_ipv6_address(host):
3042
log.warning("Selected IPv6 address for machine %s: %r", machine, host)
3043
return host
3044
3045
3046
class Controller:
3047
"""Represents the controller for a model or models."""
3048
3049
def __init__(self, name):
3050
self.name = name
3051
3052
3053
class GroupReporter:
3054
3055
def __init__(self, stream, expected):
3056
self.stream = stream
3057
self.expected = expected
3058
self.last_group = None
3059
self.ticks = 0
3060
self.wrap_offset = 0
3061
self.wrap_width = 79
3062
3063
def _write(self, string):
3064
self.stream.write(string)
3065
self.stream.flush()
3066
3067
def finish(self):
3068
if self.last_group:
3069
self._write("\n")
3070
3071
def update(self, group):
3072
if group == self.last_group:
3073
if (self.wrap_offset + self.ticks) % self.wrap_width == 0:
3074
self._write("\n")
3075
self._write("." if self.ticks or not self.wrap_offset else " .")
3076
self.ticks += 1
3077
return
3078
value_listing = []
3079
for value, entries in sorted(group.items()):
3080
if value == self.expected:
3081
continue
3082
value_listing.append('%s: %s' % (value, ', '.join(entries)))
3083
string = ' | '.join(value_listing)
3084
lead_length = len(string) + 1
3085
if self.last_group:
3086
string = "\n" + string
3087
self._write(string)
3088
self.last_group = group
3089
self.ticks = 0
3090
self.wrap_offset = lead_length if lead_length < self.wrap_width else 0
281
282
def format_listing(listing, expected, environment):
283
value_listing = []
284
for value, entries in listing.items():
285
if value == expected:
286
continue
287
value_listing.append('%s: %s' % (value, ', '.join(entries)))
288
return ('<%s> ' % environment) + ' | '.join(value_listing)
289
290
291
def check_wordpress(environment, host):
292
""""Check whether Wordpress has come up successfully.
293
294
Times out after 30 seconds.
295
"""
296
welcome_text = ('Welcome to the famous five minute WordPress'
297
' installation process!')
298
url = 'http://%s/wp-admin/install.php' % host
299
for ignored in until_timeout(30):
300
try:
301
page = urllib2.urlopen(url)
302
except (urllib2.URLError, httplib.HTTPException, socket.error):
303
pass
304
else:
305
if welcome_text in page.read():
306
break
307
# Let's not DOS wordpress
308
sleep(1)
309
else:
310
raise Exception(
311
'Cannot get welcome screen at %s %s' % (url, environment))
Loggerhead is a web-based interface for Breezy
Version: 2.0.1