1
# Copyright 2015-2019 Canonical Ltd. This software is licensed under the
2
# Copyright 2015 Canonical Ltd. This software is licensed under the
2
3
# GNU Affero General Public License version 3 (see the file LICENSE).
5
"""A script that builds a snap."""
4
7
from __future__ import print_function
9
from collections import OrderedDict
13
from optparse import OptionParser
15
from textwrap import dedent
17
from six.moves.urllib.parse import urlparse
19
from lpbuildd.target.operation import Operation
20
from lpbuildd.target.snapstore import SnapStoreOperationMixin
21
from lpbuildd.target.vcs import VCSOperationMixin
19
from urlparse import urlparse
21
from lpbuildd.util import (
24
28
RETCODE_FAILURE_INSTALL = 200
25
29
RETCODE_FAILURE_BUILD = 201
28
logger = logging.getLogger(__name__)
31
class SnapChannelsAction(argparse.Action):
33
def __init__(self, option_strings, dest, nargs=None, **kwargs):
35
raise ValueError("nargs not allowed")
36
super(SnapChannelsAction, self).__init__(
37
option_strings, dest, **kwargs)
39
def __call__(self, parser, namespace, values, option_string=None):
41
raise argparse.ArgumentError(
42
self, "'{}' is not of the form 'snap=channel'".format(values))
43
snap, channel = values.split("=", 1)
44
if getattr(namespace, self.dest, None) is None:
45
setattr(namespace, self.dest, {})
46
getattr(namespace, self.dest)[snap] = channel
49
class BuildSnap(VCSOperationMixin, SnapStoreOperationMixin, Operation):
51
description = "Build a snap."
53
core_snap_names = ["core", "core16", "core18", "core20"]
56
def add_arguments(cls, parser):
57
super(BuildSnap, cls).add_arguments(parser)
59
"--channel", action=SnapChannelsAction, metavar="SNAP=CHANNEL",
60
dest="channels", default={}, help=(
61
"install SNAP from CHANNEL "
62
"(supported snaps: {}, snapcraft)".format(
63
", ".join(cls.core_snap_names))))
66
help="ID of the request triggering this build on Launchpad")
68
"--build-request-timestamp",
69
help="RFC3339 timestamp of the Launchpad build request")
71
"--build-url", help="URL of this build on Launchpad")
72
parser.add_argument("--proxy-url", help="builder proxy url")
74
"--revocation-endpoint",
75
help="builder proxy token revocation endpoint")
77
"--build-source-tarball", default=False, action="store_true",
79
"build a tarball containing all source code, including "
80
"external dependencies"))
82
"--private", default=False, action="store_true",
83
help="build a private snap")
84
parser.add_argument("name", help="name of snap to build")
86
def __init__(self, args, parser):
87
super(BuildSnap, self).__init__(args, parser)
88
self.bin = os.path.dirname(sys.argv[0])
90
def run_build_command(self, args, env=None, **kwargs):
91
"""Run a build command in the target.
93
:param args: the command and arguments to run.
32
def get_build_path(build_id, *extra):
33
"""Generate a path within the build directory.
35
:param build_id: the build id to use.
36
:param extra: the extra path segments within the build directory.
37
:return: the generated path.
39
return os.path.join(os.environ["HOME"], "build-" + build_id, *extra)
45
def __init__(self, options, name):
46
self.options = options
48
self.chroot_path = get_build_path(
49
self.options.build_id, 'chroot-autobuild')
50
# Set to False for local testing if your chroot doesn't have an
51
# appropriate certificate for your codehosting system.
52
self.ssl_verify = True
54
def chroot(self, args, echo=False, get_output=False):
55
"""Run a command in the chroot.
57
:param args: the command and arguments to run.
58
:param echo: if True, print the command before executing it.
59
:param get_output: if True, return the output from the command.
61
args = set_personality(self.options.arch, args)
64
"Running in chroot: %s" % ' '.join(
65
"'%s'" % arg for arg in args))
67
cmd = ["/usr/bin/sudo", "/usr/sbin/chroot", self.chroot_path] + args
69
return subprocess.check_output(cmd, universal_newlines=True)
71
subprocess.check_call(cmd)
73
def run_build_command(self, args, path="/build", env=None, echo=False,
75
"""Run a build command in the chroot.
77
This is unpleasant because we need to run it in /build under sudo
78
chroot, and there's no way to do this without either a helper
79
program in the chroot or unpleasant quoting. We go for the
82
:param args: the command and arguments to run.
83
:param path: the working directory to use in the chroot.
94
84
:param env: dictionary of additional environment variables to set.
95
:param kwargs: any other keyword arguments to pass to Backend.run.
85
:param echo: if True, print the command before executing it.
86
:param get_output: if True, return the output from the command.
97
full_env = OrderedDict()
98
full_env["LANG"] = "C.UTF-8"
99
full_env["SHELL"] = "/bin/sh"
88
args = [shell_escape(arg) for arg in args]
89
path = shell_escape(path)
101
94
full_env.update(env)
102
return self.backend.run(args, env=full_env, **kwargs)
96
"%s=%s" % (key, shell_escape(value))
97
for key, value in full_env.items()] + args
98
command = "cd %s && %s" % (path, " ".join(args))
100
["/bin/sh", "-c", command], echo=echo, get_output=get_output)
104
102
def save_status(self, status):
105
103
"""Save a dictionary of status information about this build.
107
105
This will be picked up by the build manager and included in XML-RPC
108
106
status responses.
110
status_path = os.path.join(self.backend.build_path, "status")
108
status_path = get_build_path(self.options.build_id, "status")
111
109
with open("%s.tmp" % status_path, "w") as status_file:
112
110
json.dump(status, status_file)
113
111
os.rename("%s.tmp" % status_path, status_path)
115
def install_svn_servers(self):
116
proxy = urlparse(self.args.proxy_url)
117
svn_servers = dedent("""\
119
http-proxy-host = {host}
120
http-proxy-port = {port}
121
""".format(host=proxy.hostname, port=proxy.port))
122
# We should never end up with an authenticated proxy here since
123
# lpbuildd.snap deals with it, but it's almost as easy to just
124
# handle it as to assert that we don't need to.
126
svn_servers += "http-proxy-username = {}\n".format(proxy.username)
128
svn_servers += "http-proxy-password = {}\n".format(proxy.password)
129
with tempfile.NamedTemporaryFile(mode="w+") as svn_servers_file:
130
svn_servers_file.write(svn_servers)
131
svn_servers_file.flush()
132
os.fchmod(svn_servers_file.fileno(), 0o644)
133
self.backend.run(["mkdir", "-p", "/root/.subversion"])
134
self.backend.copy_in(
135
svn_servers_file.name, "/root/.subversion/servers")
137
113
def install(self):
138
logger.info("Running install phase...")
140
if self.args.backend == "lxd":
141
# udev is installed explicitly to work around
142
# https://bugs.launchpad.net/snapd/+bug/1731519.
143
for dep in "snapd", "fuse", "squashfuse", "udev":
144
if self.backend.is_package_available(dep):
146
deps.extend(self.vcs_deps)
147
if self.args.proxy_url:
148
deps.extend(["python3", "socat"])
149
if "snapcraft" in self.args.channels:
150
# snapcraft requires sudo in lots of places, but can't depend on
151
# it when installed as a snap.
114
print("Running install phase...")
116
if self.options.branch is not None:
154
deps.append("snapcraft")
155
self.backend.run(["apt-get", "-y", "install"] + deps)
156
if self.args.backend in ("lxd", "fake"):
157
self.snap_store_set_proxy()
158
for snap_name in self.core_snap_names:
159
if snap_name in self.args.channels:
162
"--channel=%s" % self.args.channels[snap_name],
164
if "snapcraft" in self.args.channels:
166
["snap", "install", "--classic",
167
"--channel=%s" % self.args.channels["snapcraft"],
169
if self.args.proxy_url:
170
self.backend.copy_in(
171
os.path.join(self.bin, "snap-git-proxy"),
172
"/usr/local/bin/snap-git-proxy")
173
self.install_svn_servers()
120
self.chroot(["apt-get", "-y", "install"] + deps)
176
123
"""Collect git or bzr branch."""
177
logger.info("Running repo phase...")
179
if self.args.proxy_url:
180
env["http_proxy"] = self.args.proxy_url
181
env["https_proxy"] = self.args.proxy_url
182
env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
183
self.vcs_fetch(self.args.name, cwd="/build", env=env)
124
print("Running repo phase...")
126
if self.options.proxy_url:
127
env["http_proxy"] = self.options.proxy_url
128
env["https_proxy"] = self.options.proxy_url
129
if self.options.branch is not None:
130
self.run_build_command(['ls', '/build'])
131
cmd = ["bzr", "branch", self.options.branch, self.name]
132
if not self.ssl_verify:
133
cmd.insert(1, "-Ossl.cert_reqs=none")
135
assert self.options.git_repository is not None
136
assert self.options.git_path is not None
138
"git", "clone", "-b", self.options.git_path,
139
self.options.git_repository, self.name,
141
if not self.ssl_verify:
142
env["GIT_SSL_NO_VERIFY"] = "1"
143
self.run_build_command(cmd, env=env)
185
if self.args.branch is not None:
145
if self.options.branch is not None:
186
146
status["revision_id"] = self.run_build_command(
188
cwd=os.path.join("/build", self.args.name),
189
get_output=True).rstrip("\n")
147
["bzr", "revno", self.name], get_output=True).rstrip("\n")
193
if self.args.git_path is not None else "HEAD")
194
149
status["revision_id"] = self.run_build_command(
195
# The ^{} suffix copes with tags: we want to peel them
196
# recursively until we get an actual commit.
197
["git", "rev-parse", rev + "^{}"],
198
cwd=os.path.join("/build", self.args.name),
150
["git", "-C", self.name, "rev-parse", self.options.git_path],
199
151
get_output=True).rstrip("\n")
200
152
self.save_status(status)
203
def image_info(self):
205
if self.args.build_request_id is not None:
206
data["build-request-id"] = 'lp-{}'.format(
207
self.args.build_request_id)
208
if self.args.build_request_timestamp is not None:
209
data["build-request-timestamp"] = self.args.build_request_timestamp
210
if self.args.build_url is not None:
211
data["build_url"] = self.args.build_url
212
return json.dumps(data, sort_keys=True)
215
155
"""Run pull phase."""
216
logger.info("Running pull phase...")
218
env["SNAPCRAFT_LOCAL_SOURCES"] = "1"
219
env["SNAPCRAFT_SETUP_CORE"] = "1"
220
if not self.args.private:
221
env["SNAPCRAFT_BUILD_INFO"] = "1"
222
env["SNAPCRAFT_IMAGE_INFO"] = self.image_info
223
env["SNAPCRAFT_BUILD_ENVIRONMENT"] = "host"
224
if self.args.proxy_url:
225
env["http_proxy"] = self.args.proxy_url
226
env["https_proxy"] = self.args.proxy_url
227
env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
156
print("Running pull phase...")
158
"SNAPCRAFT_LOCAL_SOURCES": "1",
159
"SNAPCRAFT_SETUP_CORE": "1",
161
if self.options.proxy_url:
162
env["http_proxy"] = self.options.proxy_url
163
env["https_proxy"] = self.options.proxy_url
228
164
self.run_build_command(
229
165
["snapcraft", "pull"],
230
cwd=os.path.join("/build", self.args.name),
166
path=os.path.join("/build", self.name),
232
if self.args.build_source_tarball:
233
self.run_build_command(
234
["tar", "-czf", "%s.tar.gz" % self.args.name,
235
"--format=gnu", "--sort=name", "--exclude-vcs",
236
"--numeric-owner", "--owner=0", "--group=0",
241
170
"""Run all build, stage and snap phases."""
242
logger.info("Running build phase...")
244
if not self.args.private:
245
env["SNAPCRAFT_BUILD_INFO"] = "1"
246
env["SNAPCRAFT_IMAGE_INFO"] = self.image_info
247
env["SNAPCRAFT_BUILD_ENVIRONMENT"] = "host"
248
if self.args.proxy_url:
249
env["http_proxy"] = self.args.proxy_url
250
env["https_proxy"] = self.args.proxy_url
251
env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
171
print("Running build phase...")
173
if self.options.proxy_url:
174
env["http_proxy"] = self.options.proxy_url
175
env["https_proxy"] = self.options.proxy_url
252
176
self.run_build_command(
254
cwd=os.path.join("/build", self.args.name),
261
logger.exception('Install failed')
262
return RETCODE_FAILURE_INSTALL
268
logger.exception('Build failed')
269
return RETCODE_FAILURE_BUILD
177
["snapcraft"], path=os.path.join("/build", self.name), env=env)
179
def revoke_token(self):
180
"""Revoke builder proxy token."""
181
print("Revoking proxy token...")
182
url = urlparse(self.options.proxy_url)
183
auth = '{}:{}'.format(url.username, url.password)
185
'Authorization': 'Basic {}'.format(base64.b64encode(auth))
187
req = urllib2.Request(self.options.revocation_endpoint, None, headers)
188
req.get_method = lambda: 'DELETE'
191
except (urllib2.HTTPError, urllib2.URLError) as e:
192
print('Unable to revoke token for %s: %s' % (url.username, e))
196
parser = OptionParser("%prog [options] NAME")
197
parser.add_option("--build-id", help="build identifier")
199
"--arch", metavar="ARCH", help="build for architecture ARCH")
201
"--branch", metavar="BRANCH", help="build from this Bazaar branch")
203
"--git-repository", metavar="REPOSITORY",
204
help="build from this Git repository")
206
"--git-path", metavar="REF-PATH",
207
help="build from this ref path in REPOSITORY")
208
parser.add_option("--proxy-url", help="builder proxy url")
209
parser.add_option("--revocation-endpoint",
210
help="builder proxy token revocation endpoint")
211
options, args = parser.parse_args()
212
if (options.git_repository is None) != (options.git_path is None):
214
"must provide both --git-repository and --git-path or neither")
215
if (options.branch is None) == (options.git_repository is None):
217
"must provide exactly one of --branch and --git-repository")
220
"must provide a package name and no other positional arguments")
222
builder = SnapBuilder(options, name)
226
traceback.print_exc()
227
return RETCODE_FAILURE_INSTALL
233
traceback.print_exc()
234
return RETCODE_FAILURE_BUILD
236
if options.revocation_endpoint is not None:
237
builder.revoke_token()
238
return RETCODE_SUCCESS
241
if __name__ == "__main__":
added
removed
buildsnap
