3
"""Add missing result.yaml in S3; ensue that existing files contain
7
from __future__ import print_function
8
from argparse import ArgumentParser
9
from datetime import datetime
13
from tempfile import NamedTemporaryFile
21
ARCHIVE_URL = 's3://juju-qa-data/juju-ci/products/'
22
ISO_8601_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
23
LONG_AGO = datetime(2000, 1, 1)
26
def get_ci_director_state():
27
state_file_path = os.path.join(
28
os.environ['HOME'], '.config/ci-director-state')
29
with open(state_file_path) as state_file:
30
return yaml.load(state_file)['versions']
34
text = s3_cmd(['ls', '-r', ARCHIVE_URL])
35
for line in text.strip().split('\n'):
36
file_date, file_time, size, url = re.split(r'\s+', line)
37
file_date = [int(part) for part in file_date.split('-')]
38
file_time = [int(part) for part in file_time.split(':')]
39
file_time = datetime(*(file_date + file_time))
40
revision_number, filename = re.search(
41
r'^{}version-(\d+)/(.*)$'.format(ARCHIVE_URL), url).groups()
42
yield int(revision_number), filename, file_time
45
def get_s3_revision_info():
47
for revision_number, file_name, file_time in list_s3_files():
48
revision = all_revisions.setdefault(revision_number, {
50
'artifact_time': LONG_AGO,
52
if file_name in ('result.yaml', 'result.json'):
53
# Many result.json files were added on 2014-08-14 for older
54
# builds, so we may have both a result.yaml file and a
56
revision['result'][file_time] = file_name
58
revision['artifact_time'] = max(
59
revision['artifact_time'], file_time)
60
# The most recent version may currently be building, hence a check
61
# if the result file exists is useless.
62
del all_revisions[max(all_revisions)]
63
result_file_time = revision['artifact_time']
64
for revision_number, revision_data in sorted(all_revisions.items()):
65
if not revision_data['result']:
66
result_file_name = None
68
result_file_time = min(revision_data['result'])
69
# If both a result.yaml and a result.json file exist, use
71
newer = max(revision_data['result'])
72
result_file_name = revision_data['result'][newer]
73
yield revision_number, result_file_name, result_file_time
77
ci_director_state = get_ci_director_state()
78
for revision_number, result_file, artifact_time in get_s3_revision_info():
79
state_file_result = ci_director_state.get(revision_number)
80
if state_file_result is None:
82
"Warning: No state file data available for revision",
85
if result_file is not None:
86
with temp_dir() as workspace:
87
copy_from = '{}version-{}/{}'.format(
88
ARCHIVE_URL, revision_number, result_file)
89
copy_to = os.path.join(workspace, result_file)
90
s3_cmd(['--no-progress', 'get', copy_from, copy_to])
91
with open(copy_to) as f:
92
s3_result = yaml.load(f)
93
# For paranoids: Check that the data from S3 is a subset
94
# of the data from the state file
95
s3_keys = set(s3_result)
96
state_keys = set(ci_director_state[revision_number])
97
if not s3_keys.issubset(state_keys):
99
"Warning: S3 result file for {} contains keys that do "
100
"not exist in the main state file: {}".format(
101
revision_number, s3_keys.difference(state_keys)))
103
comparable_state_data = dict(
105
for k, v in ci_director_state[revision_number].items()
107
if comparable_state_data != s3_result:
108
# This can happen when the result file was written
109
# when a -devel job is still running.
111
"Warning: Diverging data for revision {} in S3 ({}) "
112
"and in state file ({}).".format(
113
revision_number, s3_result,
114
ci_director_state[revision_number]))
115
if 'result' in s3_result:
118
if 'finished' not in state_file_result:
119
state_file_result['finished'] = artifact_time.strftime(
121
with NamedTemporaryFile() as new_result_file:
122
json.dump(state_file_result, new_result_file)
123
new_result_file.flush()
124
dest_url = '{}version-{}/result.json'.format(
125
ARCHIVE_URL, revision_number)
126
params = ['put', new_result_file.name, dest_url]
128
print(*(['s3cmd'] + params))
132
if __name__ == '__main__':
133
parser = ArgumentParser()
134
parser.add_argument('--dry-run', action='store_true')
135
args = parser.parse_args()
added
removed
add-missing-result-yaml-files.py
