2
"""Manage the blessed juju revision testing candiates."""
4
from __future__ import print_function
6
from argparse import ArgumentParser
34
def find_publish_revision_number(credentials, br_number, limit=20):
35
"""Return the publish-revsion number paired with build-revision number."""
37
job_number = 'lastSuccessfulBuild'
38
for i in range(limit):
39
build_data = get_build_data(
40
JENKINS_URL, credentials, PUBLISH_REVISION, build=job_number)
43
# Ensure we have the real job number (an int), not an alias.
44
job_number = build_data['number']
45
if get_revision_build(build_data) == str(br_number):
46
found_number = job_number
48
job_number = job_number - 1
52
def prepare_dir(dir_path, dry_run=False, verbose=False):
53
"""Create to clean a directory."""
54
if os.path.isdir(dir_path):
56
print('Cleaning %s' % dir_path)
58
shutil.rmtree(dir_path)
61
print('Creating %s' % dir_path)
66
def download_candidate_files(credentials, release_number, path, br_number,
67
pr_number=None, dry_run=False, verbose=False):
68
"""Download the files from the build-revision and publish-revision jobs.
70
The buildvars.json for the specific build-revision number is downloaded.
71
All the binary and source packages from the last successful build of
72
publish revision are downloaded.
74
artifact_dir_name = '%s-artifacts' % release_number
75
candidate_dir = os.path.join(path, artifact_dir_name)
76
prepare_dir(candidate_dir, dry_run, verbose)
78
credentials, BUILD_REVISION, br_number, 'buildvars.json',
79
candidate_dir, dry_run=dry_run, verbose=verbose)
81
pr_number = find_publish_revision_number(credentials, br_number)
83
credentials, PUBLISH_REVISION, pr_number, 'juju-core*', candidate_dir,
84
dry_run=dry_run, verbose=verbose)
87
def get_artifact_dirs(path):
88
"""List the directories that contain artifacts."""
90
for name in os.listdir(path):
91
artifacts_path = os.path.join(path, name)
92
if name.endswith('-artifacts') and os.path.isdir(artifacts_path):
97
def get_package(artifacts_path, version):
98
"""Return the path to the expected juju-core package for the localhost."""
99
release = subprocess.check_output(['lsb_release', '-sr']).strip()
100
arch = get_deb_arch()
101
package_name = 'juju-core_{}-0ubuntu1~{}.1~juju1_{}.deb'.format(
102
version, release, arch)
103
package_path = os.path.join(artifacts_path, package_name)
107
def extract_candidates(path, dry_run=False, verbose=False):
108
"""Extract all the candidate juju binaries for the local machine.
110
Each candidate will be extracted to a directory named after the version
111
the artifacts (packages) were made from. Thus the package that matches
112
the localhost's series and architecture in the master-artifacts/ directory
113
will be extracted to a sibling directory named "master/" The buildvars.json
114
data will be copied to the top of "master" to provide information about
115
the origin of the binaries.
117
for dir_name in get_artifact_dirs(path):
118
artifacts_path = os.path.join(path, dir_name)
119
buildvars_path = os.path.join(artifacts_path, 'buildvars.json')
120
with open(buildvars_path) as bf:
121
buildvars = json.load(bf)
122
version = buildvars['version']
123
package_path = get_package(artifacts_path, version)
124
candidate_path = os.path.join(path, version)
126
print('extracting %s to %s' % (package_path, candidate_path))
127
prepare_dir(candidate_path, dry_run, verbose)
129
extract_deb(package_path, candidate_path)
131
print('Copying %s to %s' % (buildvars_path, candidate_path))
133
new_path = os.path.join(candidate_path, 'buildvars.json')
134
shutil.copyfile(buildvars_path, new_path)
135
shutil.copystat(buildvars_path, new_path)
138
def get_scripts(juju_release_tools=None):
139
"""Return a tuple paths to the assemble_script and publish_scripts."""
140
assemble_script = 'assemble-streams.bash'
141
publish_script = 'publish-public-tools.bash'
142
if juju_release_tools:
143
assemble_script = os.path.join(
144
juju_release_tools, assemble_script)
145
publish_script = os.path.join(
146
juju_release_tools, publish_script)
147
return assemble_script, publish_script
150
def publish_candidates(path, streams_path,
151
juju_release_tools=None, dry_run=False, verbose=False):
152
"""Assemble and publish weekly streams from the candidates."""
153
timestamp = datetime.datetime.utcnow().strftime('%Y_%m_%dT%H_%M_%S')
154
with temp_dir() as debs_path:
155
for dir_name in get_artifact_dirs(path):
156
artifacts_path = os.path.join(path, dir_name)
157
branch_name = dir_name.split('-')[0]
158
for deb_name in os.listdir(artifacts_path):
159
deb_path = os.path.join(artifacts_path, deb_name)
161
print('Copying %s' % deb_path)
162
new_path = os.path.join(debs_path, deb_name)
163
shutil.copyfile(deb_path, new_path)
164
if deb_name == 'buildvars.json':
165
# buildvars.json is also in the artifacts_path; copied by
166
# download_candidate_files(). Set it aside so it can be
167
# sync'd to S3 as a record of what was published.
168
buildvar_dir = '{}/weekly/{}/{}'.format(
169
path, timestamp, branch_name)
170
if not os.path.isdir(buildvar_dir):
171
os.makedirs(buildvar_dir)
172
buildvar_path = '{}/{}'.format(buildvar_dir, deb_name)
173
shutil.copyfile(deb_path, buildvar_path)
174
assemble_script, publish_script = get_scripts(juju_release_tools)
175
# XXX sinzui 2014-12-01: IGNORE uses the local juju, but when
176
# testing juju's that change generate-tools, we may need to use
177
# the highest version.
179
assemble_script, '-t', debs_path, 'weekly', 'IGNORE',
181
run_command(command, dry_run=dry_run, verbose=verbose)
182
publish(streams_path, publish_script, dry_run=dry_run, verbose=verbose)
183
# Sync buildvars.json files out to s3.
184
url = 's3://juju-qa-data/juju-releases/weekly/'
185
s3_path = '{}/weekly/{}'.format(path, timestamp)
187
print('Calling s3cmd to sync %s out to %s' % (s3_path, url))
189
s3_cmd(['sync', s3_path, url])
190
extract_candidates(path, dry_run=dry_run, verbose=verbose)
193
def publish(streams_path, publish_script, dry_run=False, verbose=False):
194
juju_dist_path = os.path.join(streams_path, 'juju-dist')
195
command = [publish_script, 'weekly', juju_dist_path, 'cpc']
196
for attempt in range(3):
198
run_command(command, dry_run=dry_run, verbose=verbose)
200
except subprocess.CalledProcessError:
201
# Raise an error when the third attempt fails; the cloud is ill.
206
def parse_args(args=None):
207
"""Return the argument parser for this program."""
208
parser = ArgumentParser("Manage the successful Juju CI candidates.")
210
'-d', '--dry-run', action='store_true', default=False,
211
help='Do not make changes.')
213
'-v', '--verbose', action='store_true', default=False,
214
help='Increase verbosity.')
215
subparsers = parser.add_subparsers(help='sub-command help', dest="command")
216
# ./candidate download -b 1234 master ~/candidate
217
parser_update = subparsers.add_parser(
218
'download', help='download a candidate')
219
parser_update.add_argument(
220
'-b', '--br-number', default='lastSuccessfulBuild',
221
help="The specific build-revision number.")
222
parser_update.add_argument(
224
help="The specific publish-revision-revision number.")
225
parser_update.add_argument(
226
'release_number', help='The successfully test branch release number.')
227
parser_update.add_argument(
228
'path', help='The path to save the candiate data to.')
229
add_credential_args(parser_update)
230
# ./candidate extract master ~/candidate
231
parser_extract = subparsers.add_parser(
233
help='extract candidates that match the local series and arch.')
234
parser_extract.add_argument(
235
'path', help='The path to the candiate data dir.')
236
# ./candidate --juju-release-tools $JUJU_RELEASE_TOOLS \
237
# publish ~/candidate ~/streams
238
parser_publish = subparsers.add_parser(
239
'publish', help='Publish streams for the candidates')
240
parser_publish.add_argument(
241
'-t', '--juju-release-tools',
242
help='The path to the juju-release-tools dir.')
243
parser_publish.add_argument(
244
'path', help='The path to the candiate data dir.')
245
parser_publish.add_argument(
246
'streams_path', help='The path to the streams data dir.')
247
parsed_args = parser.parse_args(args)
248
return parsed_args, get_credentials(parsed_args)
252
"""Manage successful Juju CI candiates."""
253
args, credentials = parse_args(argv)
255
if args.command == 'download':
256
download_candidate_files(
257
credentials, args.release_number, args.path, args.br_number,
258
args.pr_number, dry_run=args.dry_run, verbose=args.verbose)
259
elif args.command == 'extract':
261
args.path, dry_run=args.dry_run, verbose=args.verbose)
262
elif args.command == 'publish':
264
args.path, args.streams_path,
265
juju_release_tools=args.juju_release_tools,
266
dry_run=args.dry_run, verbose=args.verbose)
267
except Exception as e:
270
traceback.print_tb(sys.exc_info()[2])
277
if __name__ == '__main__':
278
sys.exit(main(sys.argv[1:]))
added
removed
candidate.py
