« back to all changes in this revision
Viewing changes to cidirector/update_outcome.py
- browse files at revision 170
- compare with another revision
- download diff
- download tarball
- view history from revision 170
- Committer: Aaron Bentley
- Date: 2016-06-15 15:03:51 UTC
- mfrom: (169.1.24 update-cloud-health-2)
- Revision ID: aaron.bentley@canonical.com-20160615150351-0rmb0w5qv7632b26
Implement cloud-health in update_outcome.
added
removed
cidirector/update_outcome.py
2
2
from __future__ import print_function
3
3
4
4
from argparse import ArgumentParser
5
from datetime import datetime
5
from datetime import (
6
datetime,
7
timedelta,
8
)
6
9
import errno
7
10
import os
8
11
import logging
21
24
from jobs import (
22
25
BUILD_REVISION,
23
26
FAILURE,
27
parse_tags,
24
28
SUCCESS,
25
29
)
26
30
from storage import (
27
31
BUILDING,
32
datetime_from_timestamp,
28
33
FAILED,
29
34
find_blockers,
35
get_substrate_from_parsed,
30
36
PENDING,
31
37
SUCCEEDED,
32
38
topo_sort,
41
47
__metaclass__ = type
42
48
43
49
50
CI_DIRECTOR = 'ci-director'
51
CLOUD_HEALTH = 'cloud-health'
52
53
44
54
def parse_args():
45
55
parser = ArgumentParser()
46
56
parser.add_argument('-v', '--verbose', help='Verbose info',
52
62
return parser.parse_args()
53
63
54
64
55
def get_cid_config(job_info):
65
def get_cid_config(job_info, cloud_health=False):
66
section = CLOUD_HEALTH if cloud_health else CI_DIRECTOR
56
67
description = job_info.get('description')
57
68
return ResourcefulJob.config_from_description(
58
job_info['name'], description, logging.getLogger())
69
job_info['name'], description, logging.getLogger(), section=section)
59
70
60
71
61
72
def iter_job_builds(jenkins, last_completed):
76
87
job_info = jenkins.get_job_info(job_name)
77
88
config = get_cid_config(job_info)
78
89
if job_name != BUILD_REVISION and config is None:
79
logging.info('Skipping {}; no ci-director config'.format(
80
job_name))
81
continue
90
if get_cid_config(job_info, cloud_health=True) is None:
91
logging.info('Skipping {}; no ci-director config'.format(
92
job_name))
93
continue
82
94
builds = sorted(b['number'] for b in job_info['builds'])
83
95
builds = [b for b in builds if b > job_last]
84
96
yield job_info, builds
85
97
86
98
87
def iter_build_data(jenkins, job_name, builds):
88
"""Iterate through the build data for a job.
89
90
jenkins is a jenkins.Jenkins instance. job_name is the name of the job.
91
builds is a list of build numbers.
92
93
Yield pairs of (revision_build, build_info) for each build.
94
95
Handle "build-revision" specially; it is its own revision-build. Skip any
96
other builds without an explicit build-revision.
97
"""
99
def iter_build_info(jenkins, job_name, builds):
98
100
for build in builds:
99
101
logging.info('Scanning {} #{}'.format(job_name, build))
100
build_info = jenkins.get_build_info(job_name, build)
101
if job_name == BUILD_REVISION:
102
revision_build = build_info['number']
103
else:
104
build_params = get_build_parameters(build_info)
105
revision_build = build_params.get('revision_build', '')
106
try:
107
revision_build = int(revision_build)
108
except ValueError:
109
continue
110
yield revision_build, build_info
102
yield jenkins.get_build_info(job_name, build)
103
104
105
def get_revision_build(job_name, build_info):
106
if job_name == BUILD_REVISION:
107
return build_info['number']
108
else:
109
build_params = get_build_parameters(build_info)
110
revision_build = build_params.get('revision_build', '')
111
try:
112
return int(revision_build)
113
except ValueError:
114
return None
115
116
117
def update_job_configs(jobs_by_name, builds_by_key):
118
# Ensure config data is provided even for jobs with no builds.
119
active_jobs = set(n for n, j in jobs_by_name.items()
120
if j['color'] != 'disabled')
121
configs = {}
122
active_jobs = set()
123
for job_name, job_info in jobs_by_name.items():
124
if job_info['color'] != 'disabled':
125
active_jobs.add(job_name)
126
config = get_cid_config(job_info, cloud_health=True)
127
if config is None:
128
config = get_cid_config(job_info, cloud_health=False)
129
configs[job_name] = config
130
131
for key_jobs in builds_by_key.values():
132
# Include now-disabled jobs that have builds.
133
for job_name in active_jobs.union(key_jobs.keys()):
134
rb_job = key_jobs.setdefault(job_name, {'builds': {}})
135
rb_job['config'] = configs[job_name]
111
136
112
137
113
138
def scan_new_builds(last_completed, jenkins):
129
154
the current Jenkins state.
130
155
"""
131
156
by_revision_build = {}
157
cloud_health = {}
132
158
new_last_completed = {}
133
all_job_info = []
159
revision_build_jobs = {}
160
cloud_jobs = {}
134
161
for job_info, builds in iter_job_builds(jenkins, last_completed):
135
all_job_info.append(job_info)
162
is_cloud = bool(get_cid_config(job_info, cloud_health=True) is not
163
None)
136
164
job_name = job_info['name']
165
if is_cloud:
166
cloud_jobs[job_name] = job_info
167
else:
168
revision_build_jobs[job_name] = job_info
137
169
last_completed_build = job_info['lastCompletedBuild']
138
170
if last_completed_build is not None:
139
171
new_last_completed[job_name] = last_completed_build['number']
140
build_data = iter_build_data(jenkins, job_name, builds)
141
for revision_build, build_info in build_data:
142
by_job = by_revision_build.setdefault(revision_build, {})
172
173
build_iter = iter_build_info(jenkins, job_name, builds)
174
for build_info in build_iter:
175
if is_cloud:
176
date = datetime_from_timestamp(build_info['timestamp'])
177
date_str = date.date().isoformat()
178
by_job = cloud_health.setdefault(date_str, {})
179
else:
180
revision_build = get_revision_build(job_name, build_info)
181
if revision_build is None:
182
continue
183
by_job = by_revision_build.setdefault(revision_build, {})
143
184
by_build_number = by_job.setdefault(job_name, {})
144
185
builds = by_build_number.setdefault('builds', {})
145
186
builds[str(build_info['number'])] = build_info
146
# Ensure config data is provided even for jobs with no builds.
147
for job_info in all_job_info:
148
job_name = job_info['name']
149
config = get_cid_config(job_info)
150
for rb_jobs in by_revision_build.values():
151
if job_name not in rb_jobs and not job_info['color'] != 'disabled':
152
continue
153
rb_job = rb_jobs.setdefault(job_name, {'builds': {}})
154
rb_job['config'] = config
155
return by_revision_build, new_last_completed
187
update_job_configs(revision_build_jobs, by_revision_build)
188
update_job_configs(cloud_jobs, cloud_health)
189
return by_revision_build, cloud_health, new_last_completed
156
190
157
191
158
192
def json_dump_pretty(content, json_file):
190
224
return not self.__eq__(other)
191
225
192
226
@classmethod
193
def from_json(cls, test_id, json):
227
def from_json(cls, test_id, json, last_build_healthy):
194
228
build_numbers = []
195
229
building = False
196
230
succeeded = False
203
237
else:
204
238
config = json['config']
205
239
failure_threshold = config.get('failure_threshold', 1)
206
voting = config.get('vote', True)
240
voting_build = config.get('vote', True) and last_build_healthy
207
241
tags = config.get('tags')
208
242
prerequisite_jobs = set(config.get('requires', '').split())
209
243
for build in json['builds'].values():
224
258
status = PENDING
225
259
226
260
build_number = None if len(build_numbers) == 0 else max(build_numbers)
227
return cls(test_id, build_number, status, failure_count, voting, tags,
228
prerequisite_jobs, blocked=[])
261
return cls(test_id, build_number, status, failure_count, voting_build,
262
tags, prerequisite_jobs, blocked=[])
229
263
230
264
def get_status(self):
231
265
return self.status
238
272
"""JobSource for use with BaseResultJudge."""
239
273
240
274
@classmethod
241
def from_build_data(cls, build_data):
275
def from_build_data(cls, build_data, last_build_sickly):
242
276
"""Create OutcomeJobSource from build data.
243
277
244
278
build_data is formatted as emitted by
245
279
OutcomeState.update_revision_builds.
246
280
"""
247
jobs = [OutcomeJob.from_json(j, b) for j, b in build_data.items()]
281
jobs = [OutcomeJob.from_json(j, b, j not in last_build_sickly)
282
for j, b in build_data.items()]
248
283
for job in topo_sort(jobs):
249
284
job.blocked = find_blockers(job, jobs)
250
285
return cls(jobs)
260
295
"""Raised when attempting to write to a directory that is not locked."""
261
296
262
297
298
class BuildFiles:
299
300
def __init__(self, outcome_state, filename_template):
301
self.outcome_state = outcome_state
302
self.filename_template = filename_template
303
304
@classmethod
305
def for_revision_builds(cls, state):
306
return BuildFiles(state, 'revision-build-{}.json')
307
308
@classmethod
309
def for_cloud_health(cls, state):
310
return BuildFiles(state, 'cloud-health-{}.json')
311
312
def filename(self, file_key):
313
return self.filename_template.format(file_key)
314
315
@staticmethod
316
def merge_revision_build_data(recorded_builds, new_builds):
317
"""Merge data from new_builds into recorded_builds.
318
319
The job config data is updated only if new_builds has builds for that
320
job.
321
"""
322
for job, job_dict in new_builds.items():
323
old_job_dict = recorded_builds.setdefault(
324
job, {'builds': {}})
325
old_job_dict['builds'].update(job_dict['builds'])
326
if len(job_dict['builds']) > 0:
327
if 'config' not in job_dict:
328
print('No config'.format(job_dict))
329
continue
330
old_job_dict['config'] = job_dict['config']
331
332
def load_json(self, file_key):
333
filename = self.filename(file_key)
334
return self.outcome_state.load_json(filename)
335
336
def update_files(self, build_data):
337
for revision_build, new_builds in build_data.items():
338
recorded_builds = self.load_json(revision_build)
339
if recorded_builds is None:
340
recorded_builds = new_builds
341
else:
342
self.merge_revision_build_data(recorded_builds, new_builds)
343
filename = self.filename(revision_build)
344
self.outcome_state.write_json(filename, recorded_builds)
345
yield revision_build, recorded_builds
346
347
348
def load_cloud_health(cloud_health_builds, build_days):
349
result = {}
350
for day in build_days:
351
file_key = day.isoformat()
352
result[file_key] = cloud_health_builds.load_json(file_key)
353
return result
354
355
356
def get_substrate(config):
357
parsed_tags = parse_tags(config['tags'])
358
return get_substrate_from_parsed(parsed_tags)
359
360
361
def find_cloud_health_builds(cloud_health_builds, substrate, start, end):
362
for jobs in cloud_health_builds.values():
363
for job in jobs.values():
364
if get_substrate(job['config']) != substrate:
365
continue
366
for buildnum, build_info in job['builds'].items():
367
duration = build_info.get('duration')
368
# A build with no duration has no result, so it's not
369
# interesting anyhow.
370
if duration is None:
371
continue
372
build_start = datetime_from_timestamp(build_info['timestamp'])
373
build_end = datetime_from_timestamp(
374
build_info['timestamp'] + duration)
375
if start > build_end:
376
continue
377
if end < build_start:
378
continue
379
yield build_info
380
381
382
def get_health_match_data(by_revision_build):
383
"""Return data on the last build for matching cloud health.
384
385
Provides a dict with keys 'substrate', 'start', 'end'. 'start' and 'end'
386
are extended by an hour to include cloud health builds that ended at most
387
an hour before the revision build or started at most an hour after the
388
revision build.
389
390
Provides the days which may contain relevant cloud health as a set of
391
datetime.date objects.
392
"""
393
lb_data = {}
394
lb_days = set()
395
for revision_build, jobs in by_revision_build.items():
396
rb_data = {}
397
for job_name, job in jobs.items():
398
if len(job['builds']) == 0:
399
continue
400
if job['config'] is None:
401
continue
402
last_build = sorted(
403
job['builds'].values(), key=lambda b: b['number'])[-1]
404
duration = last_build.get('duration')
405
if duration is None:
406
continue
407
start = datetime_from_timestamp(last_build['timestamp'])
408
end = start + timedelta(milliseconds=duration)
409
start -= timedelta(hours=1)
410
lb_days.add(start.date())
411
lb_days.add(end.date())
412
rb_data[job_name] = {
413
'substrate': get_substrate(job['config']),
414
'start': start,
415
'end': end,
416
}
417
lb_data[revision_build] = rb_data
418
return lb_data, lb_days
419
420
421
def find_last_build_sickly(health_match_data, cloud_health_build_dict):
422
last_build_sickly = {}
423
for revision_build, jobs in health_match_data.items():
424
sickly_jobs = set()
425
for job, lb_info in jobs.items():
426
ch_builds = find_cloud_health_builds(
427
cloud_health_build_dict, lb_info['substrate'],
428
lb_info['start'], lb_info['end'])
429
results = set(ch_build.get('result') for ch_build in ch_builds)
430
results.discard(SUCCESS)
431
if len(results) > 0:
432
sickly_jobs.add(job)
433
last_build_sickly[revision_build] = sickly_jobs
434
return last_build_sickly
435
436
263
437
class OutcomeState:
264
438
"""Maintain information about job outcomes."""
265
439
299
473
self.last_completed_file = None
300
474
self._lock_cxt.__exit__(exc_type, exc_val, exc_tb)
301
475
302
def write_json(self, filename, content):
476
def load_json(self, relative_path):
477
try:
478
with open(self.outcome_filename(relative_path)) as build_file:
479
return json.load(build_file)
480
except IOError as e:
481
if e.errno != errno.ENOENT:
482
raise
483
return None
484
485
def write_json(self, relative_path, content, make_dir=False):
303
486
"""Write a json file relative to the outcome_dir."""
304
487
if self.last_completed_file is None:
305
488
raise NotLocked('Write with no lock')
489
filename = self.outcome_filename(relative_path)
490
if make_dir:
491
parent = os.path.dirname(filename)
492
ensure_dir(parent)
306
493
with open(filename, 'w') as json_file:
307
494
json_dump_pretty(content, json_file)
308
495
309
def revision_build_filename(self, revision_build):
310
return self.outcome_filename(
311
'revision-build-{}.json'.format(revision_build))
312
313
@staticmethod
314
def merge_revision_build_data(recorded_builds, new_builds):
315
"""Merge data from new_builds into recorded_builds.
316
317
The job config data is updated only if new_builds has builds for that
318
job.
319
"""
320
for job, job_dict in new_builds.items():
321
old_job_dict = recorded_builds.setdefault(
322
job, {'builds': {}})
323
old_job_dict['builds'].update(job_dict['builds'])
324
if len(job_dict['builds']) > 0:
325
old_job_dict['config'] = job_dict['config']
326
327
def update_revision_builds(self, by_revision_build):
328
"""Use the supplied data to update information about revision builds
329
330
New data about known builds will override existing data, but old
331
builds will otherwise be preserved.
332
"""
333
for revision_build, new_builds in by_revision_build.items():
334
filename = self.revision_build_filename(revision_build)
335
try:
336
with open(filename) as json_file:
337
recorded_builds = json.load(json_file)
338
except IOError as e:
339
if e.errno != errno.ENOENT:
340
raise
341
recorded_builds = new_builds
342
else:
343
self.merge_revision_build_data(recorded_builds, new_builds)
344
self.write_json(filename, recorded_builds)
345
yield revision_build, recorded_builds
346
347
496
def mongo_filename(self, revision_build):
348
return os.path.join(self.outcome_filename(
349
'version-{}'.format(revision_build)), 'result.json')
497
return os.path.join(
498
'version-{}'.format(revision_build), 'result.json')
350
499
351
500
@staticmethod
352
def make_mongo_data(revision_build, build_data):
501
def make_mongo_data(revision_build, build_data, last_build_sickly):
353
502
"""Make data formatted for juju-reports' mongodb.
354
503
355
504
revision_build is the revision build number.
359
508
"""
360
509
if build_data.get('build-revision', {}).get('builds', {}) == {}:
361
510
return None
362
job_source = OutcomeJobSource.from_build_data(build_data)
511
job_source = OutcomeJobSource.from_build_data(
512
build_data, last_build_sickly)
363
513
tests = {}
364
514
for ojob in job_source.get_candidate_determine_result_jobs():
365
515
tests[ojob.test_id] = {
407
557
mongo_data['finished'] = last_updated
408
558
return mongo_data
409
559
410
def write_mongo_data(self, revision_build, build_data):
560
def write_mongo_data(self, revision_build, build_data, last_build_healthy):
411
561
"""Write mongo data to disk.
412
562
413
563
The data is formatted as emitted by make_mongo_data.
417
567
418
568
If build_data has no build-revision member, do not do anything.
419
569
"""
420
mongo_data = self.make_mongo_data(revision_build, build_data)
570
mongo_data = self.make_mongo_data(revision_build, build_data,
571
last_build_healthy)
421
572
if mongo_data is None:
422
573
return
423
574
mongo_name = self.mongo_filename(revision_build)
424
mongo_dirname = os.path.dirname(mongo_name)
425
ensure_dir(mongo_dirname)
426
self.write_json(mongo_name, mongo_data)
575
self.write_json(mongo_name, mongo_data, make_dir=True)
427
576
428
577
429
578
def main():
436
585
outcome_dir = os.path.join(os.environ['HOME'],
437
586
'.config/ci-director-outcome')
438
587
with OutcomeState(outcome_dir) as state:
439
by_revision_build, new_last_completed = scan_new_builds(
588
by_revision_build, cloud_health, new_last_completed = scan_new_builds(
440
589
state.data, jenkins)
441
for revision_build, build_data in state.update_revision_builds(
442
by_revision_build):
443
state.write_mongo_data(revision_build, build_data)
590
cloud_health_builds = BuildFiles.for_cloud_health(state)
591
list(cloud_health_builds.update_files(cloud_health))
592
revision_build_files = BuildFiles.for_revision_builds(state)
593
by_all_revision_build = dict(
594
revision_build_files.update_files(by_revision_build))
595
health_match_data, build_days = get_health_match_data(
596
by_all_revision_build)
597
cloud_health_build_dict = load_cloud_health(cloud_health_builds,
598
build_days)
599
last_build_sickly = find_last_build_sickly(health_match_data,
600
cloud_health_build_dict)
601
for revision_build, build_data in by_all_revision_build.items():
602
state.write_mongo_data(revision_build, build_data,
603
last_build_sickly[revision_build])
444
604
state.data.update(new_last_completed)
445
605
446
606
Loggerhead is a web-based interface for Breezy
Version: 2.0.1