1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
|
from datetime import datetime
import errno
from json import loads
import os
import platform
import pwd
import re
from shutil import copytree
from subprocess import (
PIPE,
Popen,
)
from time import (
sleep,
time,
)
import apt
from testtools.content import text_content
from testtools.matchers import Contains
def change_logs_permissions(log_dest):
"""Change logs permissions so auto-package-testing can access them."""
ubuntu_user = pwd.getpwnam('ubuntu')
uid, gid = ubuntu_user.pw_uid, ubuntu_user.pw_gid
for root, dirs, files in os.walk(log_dest):
for d in dirs:
os.chown(os.path.join(root, d), uid, gid)
for f in files:
os.chown(os.path.join(root, f), uid, gid)
def is_dist(distname):
_, _, distid = platform.linux_distribution()
return distid == distname
def get_ubuntu_version():
return platform.linux_distribution()[1]
def is_precise():
return is_dist('precise')
def is_saucy():
return is_dist('saucy')
def run_command(args, env=None):
"""A wrapper to Popen to run commands in the command-line."""
process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
stdout, stderr = process.communicate()
return (
process.returncode,
stdout.decode('utf-8', 'replace'),
stderr.decode('utf-8', 'replace'),
)
def retries(timeout=30, delay=1):
"""Helper for retrying something, sleeping between attempts.
Yields ``(elapsed, remaining)`` tuples, giving times in seconds.
@param timeout: From now, how long to keep iterating, in seconds.
@param delay: The sleep between each iteration, in seconds.
"""
start = time()
end = start + timeout
for now in iter(time, None):
if now < end:
yield now - start, end - now
sleep(min(delay, end - now))
else:
break
def add_log_content(runner, log_file):
"""Add content of log_file as detail to the test runner."""
try:
log_file_content = open(log_file, 'rb').read().decode('utf-8')
runner.addDetail(
"[%s] File %s content" % (datetime.now(), log_file),
text_content(log_file_content))
except IOError:
runner.addDetail(
"[%s] File %s content" % (datetime.now(), log_file),
text_content("could not be read"))
def assertStartedUpstartService(runner, service_name, log_file=None):
"""Assert that the upstart service 'service_name' is running."""
# The service might be running pre-start, retry a few times
# before giving up.
expected = service_name + " start/running"
for _, _ in retries(timeout=60, delay=5):
if log_file is not None:
add_log_content(runner, log_file)
_, output, _ = run_command(["service", service_name, "status"])
if expected in output:
break
runner.addDetail(
"[%s] Last output content" % datetime.now(), text_content(output))
runner.assertThat(output, Contains(expected))
def assertCommandReturnCode(
runner, cmd, expected_output, env=None, expected_retcode=0):
"""Assert that cmd returns appropriate return code."""
if env is None:
env = os.environ.copy()
retcode, output, err = run_command(cmd, env=env)
runner.addDetail('%s stdout' % cmd, text_content(output))
runner.addDetail('%s stderr' % cmd, text_content(err))
runner.assertThat(output, Contains(expected_output))
runner.assertIs(retcode, expected_retcode)
def get_maas_version():
"""Get the version of the installed MAAS package."""
cache = apt.Cache()
pkg = cache["maas"]
return pkg.installed.version
def get_maas_revision():
"""Get the upstream revision of the installed MAAS package.
This assumes that MAAS is installed."""
version = get_maas_version()
# Try to match all the known formats in sequence.
versions_re = [
# Daily ppa format (format 1).
r'\d*\+bzr\d*(?:\+dfsg)?-0\+(\d*)\+',
# Daily ppa format (format 2).
r'\d*\+bzr\d*(?:\+dfsg)?\+(\d*)\+',
# Release format.
r'\d*\+bzr(\d*)(?:\+dfsg)?',
]
for version_re in versions_re:
match = re.search(version_re, version)
if match is not None:
return int(match.group(1))
return None
def filter_dict(original, desired_keys):
"""Take a subset of a `dict`.
:param original: A `dict` (or `OrderedDict`).
:param desired_keys: An iterable of keys that should be retained.
Passing a `dict` (or `OrderedDict`) here will raise an
`AssertionError`, to make sure that the caller is not passing
arguments in the wrong order.
:return: A `dict` that is just like `original` in type and contents,
except that it only contains those keys that are in `desired_keys`.
Those entries keep their original values; any other entries from
`original` will be left out.
"""
assert isinstance(original, dict), "Original must be a dict."
assert not isinstance(desired_keys, dict), (
"Don't pass a dict for desired_keys. See filter_dict docstring.")
desired_keys = frozenset(desired_keys)
return type(original)(
(key, value)
for key, value in original.items()
if key in desired_keys)
def pause_until_released(message):
"""Creates a file `/tmp/paused` and pauses until it is removed so we can
ssh into the machine and run any sort of test or investigations we need.
When you are done with your testing, deleting the file will allow the tests
to continue. The `message` parameter is written to the file.
"""
pause_file = '/tmp/paused'
with open(pause_file, 'w') as f:
f.write("The CI test suite is now PAUSED.\n")
f.write("To resume, please delete this file.\n")
f.write("---\n")
f.write(message)
f.write("\n")
while os.path.isfile(pause_file):
sleep(1)
def collect_logs(log_dirs, log_dest):
"""Collect logs and configs from the test run.
This method copies logs and configs from the test run to a known
location so adt/auto-package-testing can copy them out of the testbed.
"""
for log_dir in log_dirs:
try:
copytree(log_dir, os.path.join(log_dest, log_dir.lstrip('/')))
except OSError as e:
if e.errno == errno.ENOENT:
pass
# Directory does not exist: ignore and carry on.
else:
raise
def copy_logs(log_dirs):
"""Collect logs and signal to the cluster tests finished."""
# ADT_ARTIFACTS is used by adt while /var/tmp/testresults is used by
# auto-package-testing.
log_path = os.environ.get('ADT_ARTIFACTS', '/var/tmp/testresults')
log_dest = os.path.join(log_path, 'maas-logs')
collect_logs(log_dirs, log_dest)
change_logs_permissions(log_dest)
def get_machines_with_status(runner, status):
"""Retrieve from the API all nodes with `status` (or substatus)."""
# Start by listing all nodes. The output will be shown on failure, so
# even if we can filter by status, doing it client-side may provide
# more helpful debug output.
output, _ = runner._run_maas_cli(["machines", "read"])
machine_list = loads(output)
# In 1.9 we used to look at the substatus for new status's.
# However 2.0+ we simply look at the status
machines = [
machine for machine in machine_list
if status == machine['status']
]
return machines
def wait_machines(runner, num_expected, status=None):
"""Wait for `num_expected` nodes with status `status`."""
runner.addDetail(
'wait_%d_%d' % (num_expected, status),
text_content(
"Waiting for %d node(s) with status %d."
% (num_expected, status)))
filtered_list = get_machines_with_status(runner, status)
while len(filtered_list) != num_expected:
sleep(5)
filtered_list = get_machines_with_status(runner, status)
return filtered_list
def run_maas_cli(runner, profile, args):
maascli = "maas"
retcode, output, err = run_command([maascli, profile] + args)
runner.addDetail(
'retcode for %s maas %s' % (maascli, args),
text_content('%d' % retcode))
runner.addDetail(
'stdout for %s maas %s' % (maascli, args),
text_content(output))
runner.addDetail(
'stderr for %s maas %s' % (maascli, args),
text_content(err))
return (output, err)
|