656
by Daniel Watkins
Pull buildd building in as a separate codepath. |
1 |
#!/usr/bin/env python
|
2 |
import argparse |
|
3 |
import os.path |
|
4 |
import sys |
|
5 |
import time |
|
6 |
from collections import defaultdict |
|
7 |
from multiprocessing import Process |
|
8 |
from urllib import ContentTooShortError, urlretrieve |
|
9 |
||
10 |
from launchpadlib.launchpad import Launchpad |
|
11 |
from launchpadlib.credentials import UnencryptedFileCredentialStore |
|
12 |
from lazr.restfulclient.errors import NotFound |
|
13 |
||
14 |
||
15 |
LIVEFS_OWNER = '~cloud-images-release-managers' |
|
16 |
||
17 |
||
18 |
def mangle_ppa_shortcut(shortcut): |
|
19 |
ppa_shortcut = shortcut.split(":")[1] |
|
20 |
user = ppa_shortcut.split("/")[0] |
|
21 |
if (user[0] == "~"): |
|
22 |
user = user[1:] |
|
23 |
ppa_path_objs = ppa_shortcut.split("/")[1:] |
|
24 |
ppa_path = [] |
|
25 |
if (len(ppa_path_objs) < 1): |
|
26 |
ppa_path = ['ubuntu', 'ppa'] |
|
27 |
elif (len(ppa_path_objs) == 1): |
|
28 |
ppa_path.insert(0, "ubuntu") |
|
29 |
ppa_path.extend(ppa_path_objs) |
|
30 |
else: |
|
31 |
ppa_path = ppa_path_objs |
|
32 |
ppa = "~%s/%s" % (user, "/".join(ppa_path)) |
|
33 |
return ppa |
|
34 |
||
35 |
||
36 |
def _get_livefs(lp, livefs_name, suite, image_format, project): |
|
37 |
kwargs = { |
|
38 |
'distro_series': '/ubuntu/{}'.format(suite), |
|
39 |
'name': livefs_name, |
|
40 |
'owner': '/{}'.format(LIVEFS_OWNER), |
|
41 |
}
|
|
42 |
try: |
|
43 |
return lp.livefses.getByName(**kwargs) |
|
44 |
except NotFound: |
|
45 |
kwargs['metadata'] = {} |
|
46 |
return lp.livefses.new(**kwargs) |
|
47 |
||
48 |
||
49 |
def trigger_build(lp, livefs_name, suite, arch, image_format, project, |
|
50 |
archive=None, proposed=False): |
|
51 |
if archive is None: |
|
52 |
reference = 'ubuntu' |
|
53 |
elif archive.startswith('ppa:'): |
|
54 |
reference = mangle_ppa_shortcut(archive) |
|
55 |
else: |
|
56 |
raise Exception('We only support archives like ppa:person/ppa.') |
|
57 |
archive = lp.archives.getByReference(reference=reference) |
|
58 |
lfs = _get_livefs(lp, livefs_name, suite, image_format, project) |
|
59 |
metadata = {'image_format': image_format, 'project': project} |
|
60 |
build = lfs.requestBuild( |
|
717
by Ben Howard
Revert prior commit, and build Wily using Updates. Per the Launchpad team, we are invocking buildds wrong; we should be using the Updates pocket. |
61 |
pocket='Updates' if not proposed else 'Proposed', |
656
by Daniel Watkins
Pull buildd building in as a separate codepath. |
62 |
archive=archive, |
63 |
distro_arch_series='/ubuntu/{}/{}'.format(suite, arch), |
|
64 |
metadata_override=metadata) |
|
65 |
print "Build queued; track progress at {}".format(build.web_link) |
|
66 |
return build |
|
67 |
||
68 |
||
69 |
def _start_download(file_url, target_path): |
|
70 |
def perform_download(file_url, target_path, iteration=0): |
|
71 |
print('Downloading {} to {}...'.format(file_url, target_path)) |
|
72 |
try: |
|
73 |
urlretrieve(file_url, target_path) |
|
74 |
except ContentTooShortError: |
|
75 |
if iteration > 4: |
|
76 |
raise
|
|
77 |
perform_download(file_url, target_path, iteration=iteration+1) |
|
78 |
p = Process(target=perform_download, args=(file_url, target_path)) |
|
79 |
p.start() |
|
80 |
return p |
|
81 |
||
82 |
||
83 |
def download(build, target_prefix): |
|
84 |
mapping = { |
|
85 |
'.ext4': '.ext4', |
|
86 |
'.img': '.ext4', |
|
87 |
'.manifest': '.manifest', |
|
88 |
'.tar.gz': '-root.tar.gz', |
|
89 |
}
|
|
90 |
file_urls = build.getFileUrls() |
|
91 |
print("{} build completed; the following files were created:\n{}".format( |
|
92 |
build.distro_arch_series.architecture_tag, '\n'.join(file_urls))) |
|
93 |
download_processes = set() |
|
94 |
for file_url in file_urls: |
|
95 |
for suffix in mapping: |
|
96 |
if file_url.endswith(suffix): |
|
97 |
target_path = '{}{}'.format(target_prefix, mapping[suffix]) |
|
98 |
download_processes.add(_start_download(file_url, target_path)) |
|
99 |
print("Waiting for downloads to complete.") |
|
100 |
for process in download_processes: |
|
101 |
process.join() |
|
102 |
||
103 |
||
104 |
def fetch_build_log(build, target_dir): |
|
105 |
target_path = os.path.join(target_dir, 'build_log.txt.gz') |
|
106 |
process = _start_download(build.build_log_url, target_path) |
|
107 |
process.join() |
|
108 |
||
109 |
||
110 |
def main(livefs_name, suite, arch, download_directory, image_format, project, |
|
111 |
archive=None, cred_location=None, proposed=False): |
|
112 |
if not os.path.exists(download_directory): |
|
113 |
raise Exception( |
|
114 |
'Download directory {} does not exist.'.format(download_directory)) |
|
115 |
if cred_location is None: |
|
116 |
cred_location = os.path.expanduser('~/.lp_creds') |
|
117 |
print "Using creds in {}".format(cred_location) |
|
118 |
credential_store = UnencryptedFileCredentialStore(cred_location) |
|
119 |
lp = Launchpad.login_with('cpc', 'production', version='devel', |
|
120 |
credential_store=credential_store) |
|
121 |
build = trigger_build(lp, livefs_name, suite, arch, image_format, project, |
|
122 |
archive=archive, proposed=proposed) |
|
123 |
try: |
|
124 |
previous_buildstate = defaultdict(unicode) |
|
125 |
while build.buildstate != 'Successfully built': |
|
126 |
if build.buildstate != previous_buildstate: |
|
127 |
print '{} build now in "{}" state'.format( |
|
128 |
arch, build.buildstate) |
|
129 |
previous_buildstate = build.buildstate |
|
130 |
if build.buildstate in ['Cancelled build', 'Failed to build']: |
|
131 |
fetch_build_log(build, download_directory) |
|
132 |
sys.exit(1) |
|
133 |
time.sleep(10) |
|
134 |
build.lp_refresh() |
|
135 |
finally: |
|
136 |
build.lp_refresh() |
|
137 |
if build.can_be_cancelled: |
|
138 |
print 'Scheduling build cancellation...' |
|
139 |
build.cancel() |
|
140 |
fetch_build_log(build, download_directory) |
|
141 |
target_prefix = 'ubuntu-{}-core-cloudimg-{}'.format(suite, arch) |
|
142 |
target_path = os.path.join(download_directory, target_prefix) |
|
143 |
download(build, target_path) |
|
144 |
print("All downloads complete; done!") |
|
145 |
||
146 |
||
147 |
if __name__ == '__main__': |
|
148 |
parser = argparse.ArgumentParser( |
|
149 |
description="Build an Ubuntu Core tarball using Launchpad's buildds.") |
|
150 |
parser.add_argument('--download-directory', default='.', |
|
151 |
help='The directory to download files to.') |
|
152 |
parser.add_argument('--livefs-name', default='docker-ubuntu-core', |
|
153 |
help="The name of the livefs to build in.") |
|
154 |
parser.add_argument('--livefs-owner', default=LIVEFS_OWNER, |
|
155 |
help='The owner of the livefs to build in.') |
|
156 |
parser.add_argument('--suite', required=True, help="The suite to build.") |
|
157 |
parser.add_argument('--arch', required=True, help="The arch to build.") |
|
158 |
parser.add_argument('--archive', |
|
159 |
help="An (optional) archive to build the image from.") |
|
160 |
parser.add_argument('--image-format', default='plain', |
|
161 |
help='The image format to build.') |
|
162 |
parser.add_argument('--project', default='ubuntu-core', |
|
163 |
help='The project to build.') |
|
164 |
parser.add_argument('--cred-location', |
|
165 |
help='The location of the LP creds.') |
|
166 |
parser.add_argument('--proposed', action='store_true') |
|
167 |
args = parser.parse_args() |
|
168 |
main(args.livefs_name, args.suite, args.arch, args.download_directory, |
|
169 |
args.image_format, args.project, archive=args.archive, |
|
170 |
cred_location=args.cred_location, proposed=args.proposed) |