1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
|
#!/usr/bin/env python
"""Assess network health for a given deployment or bundle"""
from __future__ import print_function
import argparse
import logging
import sys
import json
import yaml
import ast
import subprocess
from jujupy import client_from_config
from deploy_stack import (
BootstrapManager,
)
from jujucharm import (
local_charm_path,
)
from utility import (
add_basic_testing_arguments,
configure_logging,
)
__metaclass__ = type
log = logging.getLogger("assess_network_health")
NO_EXPOSED_UNITS = 'No exposed units'
class ConnectionError(Exception):
"""Connection failed in some way"""
def __init__(self, message):
self.message = message
def assess_network_health(client, bundle=None, target_model=None, series=None):
"""Assesses network health for a given deployment or bundle.
:param client: The juju client in use
:param bundle: Optional bundle to test on
:param model: Optional existing model to test under
"""
setup_testing_environment(client, bundle, target_model, series)
log.info("Starting network tests")
agnostic_result = ensure_juju_agnostic_visibility(client)
log.info('Agnostic result:\n {}'.format(json.dumps(agnostic_result,
indent=4,
sort_keys=True)))
visibility_result = neighbor_visibility(client)
log.info('Visibility result:\n {}'.format(json.dumps(visibility_result,
indent=4,
sort_keys=True)))
exposed_result = ensure_exposed(client, series)
log.info('Exposure result:\n {}'.format(json.dumps(exposed_result,
indent=4,
sort_keys=True)) or
NO_EXPOSED_UNITS)
log.info('Network tests complete, parsing results.')
parse_final_results(agnostic_result, visibility_result, exposed_result)
def setup_testing_environment(client, bundle, target_model, series=None):
"""Sets up the testing environment given an option bundle and/or model.
:param client: The juju client in use
:param bundle: Optional bundle to test on or None
:param model: Optional existing model to test under
"""
log.info("Setting up test environment")
if target_model:
connect_to_existing_model(client, target_model)
if bundle:
setup_bundle_deployment(client, bundle)
elif bundle is None and target_model is None:
setup_dummy_deployment(client, series)
charm_path = local_charm_path(charm='network-health', series=series,
juju_ver=client.version)
client.deploy(charm_path)
client.wait_for_started()
client.wait_for_workloads()
applications = get_juju_status(client)['applications'].keys()
applications.remove('network-health')
log.info('Known applications: {}'.format(applications))
for app in applications:
try:
client.juju('add-relation', (app, 'network-health'))
except subprocess.CalledProcessError:
log.error('Could not relate {} & network-health'.format(app))
client.wait_for_workloads()
for app in applications:
client.wait_for_subordinate_units(app, 'network-health')
def connect_to_existing_model(client, target_model):
"""Connects to an existing Juju model.
:param client: Juju client object without bootstrapped controller
:param target_model: Model to connect to for testing
"""
log.info("Connecting to existing model: {}".format(target_model))
if client.show_model().keys()[0] is not target_model:
client.switch(target_model)
def setup_dummy_deployment(client, series):
"""Sets up a dummy test environment with 2 ubuntu charms.
:param client: Bootstrapped juju client
"""
log.info("Deploying dummy charm for basic testing")
dummy_path = local_charm_path(charm='ubuntu', series=series,
juju_ver=client.version)
client.deploy(dummy_path, num=2)
client.juju('expose', ('ubuntu',))
def setup_bundle_deployment(client, bundle):
"""Deploys a test environment with supplied bundle.
:param bundle: Path to a bundle
"""
log.info("Deploying bundle specified at {}".format(bundle))
client.deploy_bundle(bundle)
def get_juju_status(client):
"""Gets juju status dict for supplied client.
:param client: Juju client object
"""
return client.get_status().status
def ensure_juju_agnostic_visibility(client):
"""Determine if known juju machines are visible.
:param machine: List of machine IPs to test
:return: Connection attempt results
"""
log.info('Starting agnostic visibility test')
machines = get_juju_status(client)['machines']
result = {}
for machine, info in machines.items():
result[machine] = {}
for ip in info['ip-addresses']:
try:
output = subprocess.check_output("ping -c 1 " + ip, shell=True)
except subprocess.CalledProcessError, e:
log.error('Error with ping attempt to {}: {}'.format(ip, e))
result[machine][ip] = False
result[machine][ip] = True
return result
def neighbor_visibility(client):
"""Check if each application's units are visible, including our own.
:param client: The juju client in use
"""
log.info('Starting neighbor visibility test')
apps = get_juju_status(client)['applications']
targets = parse_targets(apps)
result = {}
nh_units = []
for service in apps.values():
for unit in service.get('units', {}).values():
nh_units.extend(unit.get('subordinates').keys())
for nh_unit in nh_units:
service_results = {}
for service, units in targets.items():
res = ping_units(client, nh_unit, units)
service_results[service] = ast.literal_eval(res)
result[nh_unit] = service_results
return result
def ensure_exposed(client, series):
"""Ensure exposed applications are visible from the outside.
:param client: The juju client in use
:return: Exposure test results in dict by pass/fail
"""
log.info('Starting test of exposed units')
apps = get_juju_status(client)['applications']
targets = parse_targets(apps)
exposed = [app for app, e in apps.items() if e.get('exposed') is True]
if len(exposed) is 0:
log.info('No exposed units, aboring test.')
return None
new_client = setup_expose_test(client, series)
service_results = {}
for service, units in targets.items():
service_results[service] = ping_units(new_client, 'network-health/0',
units)
log.info(service_results)
return parse_expose_results(service_results, exposed)
def setup_expose_test(client, series):
"""Sets up new model to run exposure test.
:param client: The juju client in use
:return: New juju client object
"""
new_client = client.add_model('exposetest')
dummy_path = local_charm_path(charm='ubuntu', series=series,
juju_ver=client.version)
new_client.deploy(dummy_path)
charm_path = local_charm_path(charm='network-health', series=series,
juju_ver=client.version)
new_client.deploy(charm_path)
new_client.wait_for_started()
new_client.wait_for_workloads()
new_client.juju('add-relation', ('ubuntu', 'network-health'))
new_client.wait_for_subordinate_units('ubuntu', 'network-health')
return new_client
def parse_expose_results(service_results, exposed):
"""Parses expose test results into dict of pass/fail.
:param service_results: Raw results from expose test
:return: Parsed results dict
"""
result = {'fail': (),
'pass': ()}
for service, results in service_results.items():
# If we could connect but shouldn't, fail
if 'True' in results and service not in exposed:
result['fail'] = result['fail'] + (service,)
# If we could connect but should, pass
elif 'True' in results and service in exposed:
result['pass'] = result['pass'] + (service,)
# If we couldn't connect and shouldn't, pass
elif 'False' in results and service not in exposed:
result['pass'] = result['pass'] + (service,)
else:
result['fail'] = result['fail'] + (service,)
return result
def parse_final_results(agnostic, visibility, exposed=None):
"""Parses test results and raises an error if any failed.
:param agnostic: Agnostic test result
:param visibility: Visibility test result
:param exposed: Exposure test result
"""
error_string = []
for machine, machine_result in agnostic.items():
for ip, res in machine_result.items():
if res is False:
error = ('Failed to ping machine {0} '
'at address {1}\n'.format(machine, ip))
error_string.append(error)
for nh_source, service_result in visibility.items():
for service, unit_res in service_result.items():
if False in unit_res.values():
failed = [u for u, r in unit_res.items() if r is False]
error = ('NH-Unit {0} failed to contact '
'unit(s): {1}\n'.format(nh_source, failed))
error_string.append(error)
if exposed and exposed['fail'] is not ():
error = ('Application(s) {0} failed expose '
'test\n'.format(exposed['fail']))
error_string.append(error)
if error_string:
raise ConnectionError('\n'.join(error_string))
return
def ping_units(client, source, units):
"""Calls out to our subordinate network-health charm to ping targets.
:param client: The juju client to address
:param source: The specific network-health unit to send from
:param units: The units to ping
"""
units = to_json(units)
args = "targets='{}'".format(units)
retval = client.action_do(source, 'ping', args)
result = client.action_fetch(retval)
result = yaml.safe_load(result)['results']['results']
return result
def to_json(units):
"""Returns a formatted json string to be passed through juju run-action.
:param units: Dict of units
:return: A "JSON-like" string that can be passed to Juju without it puking
"""
json_string = json.dumps(units, separators=(',', '='))
# Replace curly brackets so juju doesn't think it's YAML or JSON and puke
json_string = json_string.replace('{', '(')
json_string = json_string.replace('}', ')')
return json_string
def parse_targets(apps):
"""Returns targets based on supplied juju status information.
:param apps: Dict of applications via 'juju status --format yaml'
"""
targets = {}
for app, units in apps.items():
target_units = {}
if 'units' in units:
for unit_id, info in units.get('units').items():
target_units[unit_id] = info['public-address']
targets[app] = target_units
return targets
def parse_args(argv):
"""Parse all arguments."""
parser = argparse.ArgumentParser(description="Test Network Health")
add_basic_testing_arguments(parser)
parser.add_argument('--bundle', help='Bundle to test network against')
parser.add_argument('--model', help='Existing Juju model to test under')
parser.set_defaults(series='trusty')
return parser.parse_args(argv)
def main(argv=None):
args = parse_args(argv)
configure_logging(args.verbose)
if args.model is None:
bs_manager = BootstrapManager.from_args(args)
with bs_manager.booted_context(args.upload_tools):
assess_network_health(bs_manager.client, bundle=args.bundle,
series=args.series)
else:
client = client_from_config(args.env, args.juju_bin)
assess_network_health(client, bundle=args.bundle,
target_model=args.model, series=args.series)
return 0
if __name__ == '__main__':
sys.exit(main())
|