2
"""Access Juju CI artifacts and data."""
4
from __future__ import print_function
6
from argparse import ArgumentParser
8
from collections import namedtuple
18
from deploy_stack import destroy_environment
19
from jujuconfig import NoSuchEnvironment
25
from lsb_release import get_distro_information
27
def get_distro_information():
28
raise NotImplementedError('Not supported on this platform!')
41
JENKINS_URL = 'http://juju-ci.vapour.ws:8080'
42
BUILD_REVISION = 'build-revision'
43
PUBLISH_REVISION = 'publish-revision'
44
CERTIFY_UBUNTU_PACKAGES = 'certify-ubuntu-packages'
46
Artifact = namedtuple('Artifact', ['file_name', 'location'])
49
Credentials = namedtuple('Credentials', ['user', 'password'])
52
class CredentialsMissing(Exception):
53
"""Raised when no credentials are supplied."""
56
def get_jenkins_json(credentials, url):
57
req = urllib2.Request(url)
58
encoded = base64.encodestring(
59
'{}:{}'.format(*credentials)).replace('\n', '')
60
req.add_header('Authorization', 'Basic {}'.format(encoded))
61
build_data = urllib2.urlopen(req)
62
return json.load(build_data)
65
def get_build_data(jenkins_url, credentials, job_name,
66
build='lastSuccessfulBuild'):
67
"""Return a dict of the build data for a job build number."""
68
url = '%s/job/%s/%s/api/json' % (jenkins_url, job_name, build)
69
return get_jenkins_json(credentials, url)
72
def get_job_data(jenkins_url, credentials, job_name):
73
"""Return a dict of the job data for a job name."""
74
url = '%s/job/%s/api/json' % (jenkins_url, job_name)
75
return get_jenkins_json(credentials, url)
78
def make_artifact(build_data, artifact):
79
location = '%sartifact/%s' % (build_data['url'], artifact['relativePath'])
80
return Artifact(artifact['fileName'], location)
83
def find_artifacts(build_data, glob='*'):
85
for artifact in build_data['artifacts']:
86
file_name = artifact['fileName']
87
if fnmatch.fnmatch(file_name, glob):
88
found.append(make_artifact(build_data, artifact))
92
def list_artifacts(credentials, job_name, build, glob, verbose=False):
93
build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
94
artifacts = find_artifacts(build_data, glob)
95
for artifact in artifacts:
97
print_now(artifact.location)
99
print_now(artifact.file_name)
102
def retrieve_artifact(credentials, url, local_path):
103
auth_location = url.replace('http://',
104
'http://{}:{}@'.format(*credentials))
105
urllib.urlretrieve(auth_location, local_path)
108
def get_juju_bin_artifact(package_namer, version, build_data):
109
file_name = package_namer.get_release_package(version)
110
return get_filename_artifact(file_name, build_data)
113
def get_filename_artifact(file_name, build_data):
114
by_filename = dict((a['fileName'], a) for a in build_data['artifacts'])
115
bin_artifact = by_filename[file_name]
116
return make_artifact(build_data, bin_artifact)
119
def retrieve_buildvars(credentials, build_number):
120
build_data = get_build_data(JENKINS_URL, credentials, BUILD_REVISION,
122
artifact = get_filename_artifact('buildvars.json', build_data)
123
return get_jenkins_json(credentials, artifact.location)
126
def get_buildvars(credentials, build_number, summary=False, env='unknown',
127
revision_build=False, version=False, branch=False,
128
short_branch=False, revision=False, short_revision=False):
129
"""Return requested information as text.
131
The 'summary' kwarg returns intelligible information about the build.
132
'env' is included in the summary text. Otherwise, a space-separated
133
string composed of the each True kwarg is returned in this order:
134
revision_build version short_branch short_revision branch revision
135
Note that revision_build is always the same as build number;
136
build_number can be an alias like 'lastBuild' or 'lastSuccessfulBuild'.
138
buildvars = retrieve_buildvars(credentials, build_number)
139
buildvars['short_revision_id'] = buildvars['revision_id'][0:7]
140
buildvars['short_branch'] = buildvars['branch'].split(':')[1]
141
buildvars['env'] = env
143
'Testing {branch} {short_revision_id} on {env} for {revision_build}')
145
text = template.format(**buildvars)
149
data.append(buildvars['revision_build'])
151
data.append(buildvars['version'])
153
data.append(buildvars['short_branch'])
155
data.append(buildvars['short_revision_id'])
157
data.append(buildvars['branch'])
159
data.append(buildvars['revision_id'])
160
text = ' '.join(data)
164
def get_release_package_filename(credentials, build_data):
165
revision_build = get_revision_build(build_data)
166
version = retrieve_buildvars(credentials, revision_build)['version']
167
return PackageNamer.factory().get_release_package(version)
170
def get_juju_bin(credentials, workspace):
171
binary_job = JobNamer.factory().get_build_binary_job()
172
build_data = get_build_data(JENKINS_URL, credentials, binary_job,
174
file_name = get_release_package_filename(credentials, build_data)
175
return get_juju_binary(credentials, file_name, build_data, workspace)
178
def get_certification_bin(credentials, version, workspace):
179
build_data = get_build_data(JENKINS_URL, credentials,
180
CERTIFY_UBUNTU_PACKAGES, 'lastBuild')
181
file_name = PackageNamer.factory().get_certification_package(version)
182
return get_juju_binary(credentials, file_name, build_data, workspace)
185
def get_juju_binary(credentials, file_name, build_data, workspace):
186
artifact = get_filename_artifact(file_name, build_data)
187
target_path = os.path.join(workspace, artifact.file_name)
188
retrieve_artifact(credentials, artifact.location, target_path)
189
return acquire_binary(target_path, workspace)
192
def acquire_binary(package_path, workspace):
193
bin_dir = os.path.join(workspace, 'extracted-bin')
194
extract_deb(package_path, bin_dir)
195
for root, dirs, files in os.walk(bin_dir):
197
return os.path.join(root, 'juju')
200
def get_artifacts(credentials, job_name, build, glob, path,
201
archive=False, dry_run=False, verbose=False):
202
full_path = os.path.expanduser(path)
205
print_now('Cleaning %s' % full_path)
206
if not os.path.isdir(full_path):
207
raise ValueError('%s does not exist' % full_path)
208
shutil.rmtree(full_path)
209
os.makedirs(full_path)
210
build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
211
artifacts = find_artifacts(build_data, glob)
212
for artifact in artifacts:
213
local_path = os.path.abspath(
214
os.path.join(full_path, artifact.file_name))
216
print_now('Retrieving %s => %s' % (artifact.location, local_path))
218
print_now(artifact.file_name)
220
retrieve_artifact(credentials, artifact.location, local_path)
224
def clean_environment(env_name, verbose=False):
226
env = SimpleEnvironment.from_config(env_name)
227
except NoSuchEnvironment as e:
232
client = EnvJujuClient.by_version(env)
234
print_now("Destroying %s" % env_name)
235
destroy_environment(client, env_name)
239
def setup_workspace(workspace_dir, env=None, dry_run=False, verbose=False):
240
"""Clean the workspace directory and create an artifacts sub directory."""
241
for root, dirs, files in os.walk(workspace_dir):
243
print_now('Removing %s' % name)
245
os.remove(os.path.join(root, name))
247
print_now('Removing %s' % name)
249
shutil.rmtree(os.path.join(root, name))
250
artifacts_path = os.path.join(workspace_dir, 'artifacts')
251
print_now('Creating artifacts dir.')
253
os.mkdir(artifacts_path)
254
# "touch empty" to convince jenkins there is an archive.
255
empty_path = os.path.join(artifacts_path, 'empty')
257
with open(empty_path, 'w'):
259
if env is not None and not dry_run:
260
clean_environment(env, verbose=verbose)
263
def add_artifacts(workspace_dir, globs, dry_run=False, verbose=False):
264
"""Find files beneath the workspace_dir and move them to the artifacts.
266
The list of globs can match the full file name, part of a name, or
267
a sub directory: eg: buildvars.json, *.deb, tmp/*.deb.
269
workspace_dir = os.path.realpath(workspace_dir)
270
artifacts_dir = os.path.join(workspace_dir, 'artifacts')
271
for root, dirs, files in os.walk(workspace_dir):
272
# create a pseudo-relative path to make glob matches easy.
273
relative = os.path.relpath(root, workspace_dir)
276
if 'artifacts' in dirs:
277
dirs.remove('artifacts')
278
for file_name in files:
279
file_path = os.path.join(root, file_name)
280
file_relative_path = os.path.join(relative, file_name)
282
if fnmatch.fnmatch(file_relative_path, glob):
284
print_now("Adding artifact %s" % file_relative_path)
286
shutil.move(file_path, artifacts_dir)
290
def add_build_job_glob(parser):
291
"""Added the --build, job, and glob arguments to the parser."""
293
'-b', '--build', default='lastSuccessfulBuild',
294
help="The specific build to examine (default: lastSuccessfulBuild).")
296
'job', help="The job that collected the artifacts.")
298
'glob', nargs='?', default='*',
299
help="The glob pattern to match artifact file names.")
302
def add_credential_args(parser):
304
'--user', default=os.environ.get('JENKINS_USER'))
306
'--password', default=os.environ.get('JENKINS_PASSWORD'))
309
def parse_args(args=None):
310
"""Return the argument parser for this program."""
311
parser = ArgumentParser("List and get artifacts from Juju CI.")
313
'-d', '--dry-run', action='store_true', default=False,
314
help='Do not make changes.')
316
'-v', '--verbose', action='store_true', default=False,
317
help='Increase verbosity.')
318
subparsers = parser.add_subparsers(help='sub-command help', dest="command")
319
parser_list = subparsers.add_parser(
320
'list', help='list artifacts for a job build')
321
add_build_job_glob(parser_list)
322
parser_get = subparsers.add_parser(
323
'get', help='get artifacts for a job build')
324
add_build_job_glob(parser_get)
325
parser_get.add_argument(
326
'-a', '--archive', action='store_true', default=False,
327
help='Ensure the download path exists and remove older files.')
328
parser_get.add_argument(
329
'path', nargs='?', default='.',
330
help="The path to download the files to.")
331
add_credential_args(parser_list)
332
add_credential_args(parser_get)
333
parser_workspace = subparsers.add_parser(
334
'setup-workspace', help='Setup and clean a workspace for building.')
335
parser_workspace.add_argument(
336
'-e', '--clean-env', dest='clean_env', default=None,
337
help='Ensure the env resources are freed or deleted.')
338
parser_workspace.add_argument(
339
'path', help="The path to the existing workspace directory.")
340
parser_get_juju_bin = subparsers.add_parser(
341
'get-juju-bin', help='Retrieve and extract juju binaries.')
342
parser_get_juju_bin.add_argument('workspace', nargs='?', default='.',
343
help='The place to store binaries.')
344
add_credential_args(parser_get_juju_bin)
345
parser_get_certification_bin = subparsers.add_parser(
346
'get-certification-bin',
347
help='Retrieve and extract juju binaries for certification.')
348
parser_get_certification_bin.add_argument(
349
'version', help='The version to get certification for.')
350
parser_get_certification_bin.add_argument(
351
'workspace', nargs='?', default='.',
352
help='The place to store binaries.')
353
add_credential_args(parser_get_certification_bin)
354
parser_get_buildvars = subparsers.add_parser(
356
help='Retrieve the build-vars for a build-revision.')
357
parser_get_buildvars.add_argument(
358
'--env', default='Unknown',
359
help='The env name to include in the summary')
360
parser_get_buildvars.add_argument(
361
'--summary', action='store_true', default=False,
362
help='Summarise the build var test data')
363
parser_get_buildvars.add_argument(
364
'--revision-build', action='store_true', default=False,
365
help='Print the test revision build number')
366
parser_get_buildvars.add_argument(
367
'--version', action='store_true', default=False,
368
help='Print the test juju version')
369
parser_get_buildvars.add_argument(
370
'--short-branch', action='store_true', default=False,
371
help='Print the short name of the branch')
372
parser_get_buildvars.add_argument(
373
'--short-revision', action='store_true', default=False,
374
help='Print the short revision of the branch')
375
parser_get_buildvars.add_argument(
376
'--branch', action='store_true', default=False,
377
help='Print the test branch')
378
parser_get_buildvars.add_argument(
379
'--revision', action='store_true', default=False,
380
help='Print the test revision')
381
parser_get_buildvars.add_argument(
382
'build', help='The build-revision build number')
383
add_credential_args(parser_get_buildvars)
384
parser_get_package_name = subparsers.add_parser(
386
help='Determine the package name for the current machine.')
387
parser_get_package_name.add_argument('version', help='The version to use.')
388
parsed_args = parser.parse_args(args)
389
if parsed_args.command == 'get-build-vars' and True not in (
390
parsed_args.summary, parsed_args.revision_build,
391
parsed_args.version, parsed_args.revision,
392
parsed_args.short_branch, parsed_args.short_revision,
393
parsed_args.branch, parsed_args.revision):
394
parser_get_buildvars.error(
395
'Expected --summary or one or more of: --revision-build, '
396
'--version, --revision, --short-branch, --short-revision, '
397
'--branch, --revision')
398
credentials = get_credentials(parsed_args)
399
return parsed_args, credentials
402
def get_credentials(args):
403
if 'user' not in args:
405
if None in (args.user, args.password):
406
raise CredentialsMissing(
407
'Jenkins username and/or password not supplied.')
409
return Credentials(args.user, args.password)
413
"""A base class that has distro and arch info used to name things."""
417
dist_info = get_distro_information()
418
return cls(get_deb_arch(), dist_info['RELEASE'], dist_info['CODENAME'])
420
def __init__(self, arch, distro_release, distro_series):
422
self.distro_release = distro_release
423
self.distro_series = distro_series
426
class PackageNamer(Namer):
427
"""A class knows the names of packages."""
429
def get_release_package_suffix(self):
430
return '-0ubuntu1~{distro_release}.1~juju1_{arch}.deb'.format(
431
distro_release=self.distro_release, arch=self.arch)
433
def get_release_package(self, version):
435
'juju-core_{version}{suffix}'
436
).format(version=version, suffix=self.get_release_package_suffix())
438
def get_certification_package(self, version):
440
'juju-core_{version}~{distro_release}.1_{arch}.deb'
441
).format(version=version, distro_release=self.distro_release,
445
class JobNamer(Namer):
446
"""A class knows the names of jobs."""
448
def get_build_binary_job(self):
449
return 'build-binary-{distro_series}-{arch}'.format(
450
distro_series=self.distro_series, arch=self.arch)
454
"""Manage list and get files from Juju CI builds."""
456
args, credentials = parse_args(argv)
457
except CredentialsMissing as e:
461
if args.command == 'list':
463
credentials, args.job, args.build, args.glob,
464
verbose=args.verbose)
465
elif args.command == 'get':
467
credentials, args.job, args.build, args.glob, args.path,
468
archive=args.archive, dry_run=args.dry_run,
469
verbose=args.verbose)
470
elif args.command == 'setup-workspace':
472
args.path, env=args.clean_env,
473
dry_run=args.dry_run, verbose=args.verbose)
474
elif args.command == 'get-juju-bin':
475
print_now(get_juju_bin(credentials, args.workspace))
476
elif args.command == 'get-certification-bin':
477
path = get_certification_bin(credentials, args.version,
480
elif args.command == 'get-build-vars':
481
text = get_buildvars(
482
credentials, args.build, env=args.env,
483
summary=args.summary, revision_build=args.revision_build,
484
version=args.version, short_branch=args.short_branch,
485
short_revision=args.short_revision,
486
branch=args.branch, revision=args.revision)
488
elif args.command == 'get-package-name':
489
print_now(PackageNamer.factory().get_release_package(args.version))
491
except Exception as e:
494
traceback.print_tb(sys.exc_info()[2])
501
if __name__ == '__main__':
502
sys.exit(main(sys.argv[1:]))
added
removed
jujuci.py
