2
# Copyright (C) 2016 Canonical Ltd.
3
# Authors: Colin Watson <cjwatson@ubuntu.com>,
4
# Brian Murray <brian@ubuntu.com>
5
# Michael Vogt <mvo@ubuntu.com>
6
# Benjamin Zeller <benjamin.zeller@canonical.com>
7
# Zoltán Balogh <zoltan.balogh@canonical.com>
9
# This program is free software: you can redistribute it and/or modify
10
# it under the terms of the GNU General Public License as published by
11
# the Free Software Foundation; version 3 of the License.
13
# This program is distributed in the hope that it will be useful,
14
# but WITHOUT ANY WARRANTY; without even the implied warranty of
15
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
# GNU General Public License for more details.
18
# You should have received a copy of the GNU General Public License
19
# along with this program. If not, see <http://www.gnu.org/licenses/>.
21
from __future__ import print_function
24
from urllib.error import URLError
25
from urllib.request import urlopen
27
from urllib2 import URLError, urlopen
38
from textwrap import dedent
39
from xml.etree import ElementTree
42
from time import gmtime, strftime
46
"ubuntu-sdk-15.04-html": "ubuntu-sdk-15.04",
47
"ubuntu-sdk-15.04-papi": "ubuntu-sdk-15.04",
48
"ubuntu-sdk-15.04-qml": "ubuntu-sdk-15.04",
50
"ubuntu-sdk-15.10-html-dev1": "ubuntu-sdk-15.10-dev1",
51
"ubuntu-sdk-15.10-papi-dev1": "ubuntu-sdk-15.10-dev1",
52
"ubuntu-sdk-15.10-qml-dev1": "ubuntu-sdk-15.10-dev1",
57
"ubuntu-sdk-15.04": "vivid",
58
"ubuntu-sdk-15.10": "wily",
59
"ubuntu-sdk-16.04": "xenial",
60
"ubuntu-sdk-16.10": "yakketi",
65
"ubuntu-sdk-libs:{TARGET}",
66
"ubuntu-sdk-libs-dev:{TARGET}",
67
"ubuntu-sdk-libs-tools",
68
"oxideqt-codecs-extra",
69
"ubuntu-html5-container:{TARGET}",
73
"ubuntu-sdk-15.10-dev1": [
74
"ubuntu-sdk-libs:{TARGET}",
75
"ubuntu-sdk-libs-dev:{TARGET}",
76
"ubuntu-sdk-libs-tools",
77
"oxideqt-codecs-extra",
87
# all runtime only packages, they are only installed on NON cross chroots
94
metadata_template = """\
96
creation_date: {CREATETIME}
99
description: {FULLNAME}
106
template: hostname.tpl
115
network_settings = """\
116
# This file describes the network interfaces available on your system
117
# and how to activate them. For more information, see interfaces(5).
119
# The loopback network interface
121
iface lo inet loopback
127
primary_arches = ["amd64", "i386"]
137
# list of target architecures supported by host architectures
138
compatible_arches = {
139
"i386": ["i386", "armhf"],
140
"amd64": ["i386", "amd64", "armhf"],
144
non_meta_re = re.compile(r'^[a-zA-Z0-9+,./:=@_-]+$')
145
GEOIP_SERVER = "http://geoip.ubuntu.com/lookup"
146
overlay_ppa = "ci-train-ppa-service/stable-phone-overlay"
148
qemu_arm_static = "/usr/bin/qemu-arm-static"
150
template_directory = "templates"
151
hostname_tpl_filename = "hostname.tpl"
152
hostname_tpl = "{{ container.name }}"
153
hosts_tpl_filename = "hosts.tpl"
154
hosts_tpl = "127.0.0.1 localhost\
155
\n127.0.1.1 {{ container.name }}\
156
\n# The following lines are desirable for IPv6 capable hosts\
157
\n::1 ip6-localhost ip6-loopback\
158
\nfe00::0 ip6-localnet\
159
\nff00::0 ip6-mcastprefix\
160
\nff02::1 ip6-allnodes\
161
\nff02::2 ip6-allrouters"
162
upstart_override_tpl_filename = "upstart-override.tpl"
163
upstart_override_tpl = "manual"
183
def make_md5_sha256(file_name, combined_hash_sha256):
184
hash_md5 = hashlib.md5()
185
hash_sha256 = hashlib.sha256()
186
with open(file_name, "rb") as f:
187
for chunk in iter(lambda: f.read(65536), b""):
188
hash_sha256.update(chunk)
189
hash_md5.update(chunk)
190
combined_hash_sha256.update(chunk)
191
return (hash_md5.hexdigest(),
192
hash_sha256.hexdigest(),
193
combined_hash_sha256.hexdigest())
196
def add_image_to_download(framework, base_arch, target_arch, image_type):
197
match = re.search('ubuntu-sdk-(.*)', framework)
199
ubuntu_version = match.group(1)
200
series = framework_series[framework]
201
root_image = "%s-%s-%s-root.tar.xz" % (framework, base_arch, target_arch)
202
lxd_image = "%s-%s-%s-lxd.tar.xz" % (framework, base_arch, target_arch)
203
if not os.path.isfile(root_image):
204
print("The %s root file does not exist" % root_image)
206
if not os.path.isfile(lxd_image):
207
print("The %s lxd file does not exist" % lxd_image)
209
str_time = strftime("%a, %d %b %Y %H:%M:%S %z", gmtime())
210
short_date = strftime("%Y%m%d", gmtime())
211
combined_hash_sha256 = hashlib.sha256()
214
combined_sha256 = make_md5_sha256(lxd_image, combined_hash_sha256)
217
combined_sha256 = make_md5_sha256(root_image, combined_hash_sha256)
218
download_json = "com.ubuntu.sdkimage:released:download.json"
219
index_json = "index.json"
220
license = "http://www.canonical.com/intellectual-property-policy"
221
image_category = "com.ubuntu.sdk-image:released:download"
222
download_json_path = "streams/v1/%s" % download_json
223
download_template = '{"content_id" : \
224
"com.ubuntu.sdk-image:released:download",\
225
"datatype" : "image-downloads",\
226
"format" : "products:1.0",\
229
"updated" : "%s"}' % (license, str_time)
230
index_template = '{"index": {\
231
"com.ubuntu.sdk-image:released:download": {\
232
"datatype": "image-downloads",\
236
"format": "products:1.0"\
240
"format": "index:1.0"\
241
}' % (download_json_path, str_time, str_time)
242
if os.path.isfile(download_json):
243
with open(download_json) as download_json_file:
244
download_data = json.load(download_json_file)
246
download_data = json.loads(download_template)
247
if os.path.isfile(index_json):
248
with open(index_json) as index_json_file:
249
index_data = json.load(index_json_file)
251
index_data = json.loads(index_template)
252
products = download_data["products"]
253
pruduct_id = "com.ubuntu.sdkimage:builder:%s:%s:%s"\
254
% (base_arch, ubuntu_version, target_arch)
255
if pruduct_id not in index_data["index"][image_category]["products"]:
256
index_data["index"][image_category]["products"].append(pruduct_id)
257
lxd_file = '{"size": %s,\
258
"ftype": "lxd.tar.xz",\
259
"path": "releases/%s/%s",\
260
"combined_sha256": "%s",\
264
% (os.path.getsize(lxd_image),
270
root_file = '{"size": %s,\
271
"ftype": "root.tar.xz",\
272
"path": "releases/%s/%s",\
273
"combined_sha256": "%s",\
276
}' % (os.path.getsize(root_image),
282
items = '{"lxd.tar.xz": %s,\
284
}' % (lxd_file, root_file)
285
date_section = '{"items" : %s,\
286
"pubname": "ubuntu-%s-%s-overlay-%s-builder-%s",\
294
versions = '{"%s": %s\
295
}' % (short_date, date_section)
296
product_object = '{"versions": %s,\
299
"release_title": "%s",\
300
"release_codename": "%s",\
304
"%s-%s-%s-%s,ubuntu-framework-%s-%s-%s-%s",\
306
"support_eol": "2019-04-17"\
321
products_string = '{"%s": %s}' % (pruduct_id, product_object)
322
new_product = json.loads(products_string)
323
merged_products = dict(products, **new_product)
324
download_data["products"] = merged_products
325
download_data["updated"] = str_time
326
with open(download_json, 'w') as outfile:
327
json.dump(download_data, outfile, indent=4)
328
index_data["updated"] = str_time
329
index_data["index"][image_category]["updated"] = str_time
330
with open(index_json, 'w') as outfile:
331
json.dump(index_data, outfile, indent=4)
334
def get_geoip_country_code_prefix():
335
click_no_local_mirror = os.environ.get('CLICK_NO_LOCAL_MIRROR', 'auto')
336
if click_no_local_mirror == '1':
339
with urlopen(GEOIP_SERVER) as f:
341
et = ElementTree.fromstring(xml_data)
342
cc = et.find("CountryCode")
345
return cc.text.lower() + "."
346
except (ElementTree.ParseError, URLError):
351
def generate_sources(series, native_arch, target_arch,
352
archive_mirror, ports_mirror, components):
353
"""Generate a list of strings for apts sources.list.
355
series -- the distro series (e.g. vivid)
356
native_arch -- the native architecture (e.g. amd64)
357
target_arch -- the target architecture (e.g. armhf)
358
archive_mirror -- main mirror, e.g. http://archive.ubuntu.com/ubuntu
359
ports_mirror -- ports mirror, e.g. http://ports.ubuntu.com/ubuntu-ports
360
components -- the components as string, e.g. "main restricted universe"
362
pockets = ['%s' % series]
363
for pocket in ['updates', 'security']:
364
pockets.append('%s-%s' % (series, pocket))
367
arches = [target_arch]
368
if native_arch != target_arch:
369
arches.append(native_arch)
371
if arch not in primary_arches:
372
mirror = ports_mirror
374
mirror = archive_mirror
375
for pocket in pockets:
376
sources.append("deb [arch=%s] %s %s %s" %
377
(arch, mirror, pocket, components))
379
for pocket in pockets:
380
sources.append("deb-src %s %s %s" %
381
(archive_mirror, pocket, components))
385
def shell_escape(command):
388
if non_meta_re.match(arg):
391
escaped.append("'%s'" % arg.replace("'", "'\\''"))
392
return " ".join(escaped)
395
def strip_dev_series_from_framework(framework):
396
"""Remove trailing -dev[0-9]+ from a framework name"""
397
return re.sub(r'^(.*)-dev[0-9]+$', r'\1', framework)
402
DAEMON_POLICY = dedent("""\
421
self.target_arch = target_arch
422
self.framework = strip_dev_series_from_framework(framework)
423
self.image_type = image_type
425
series = framework_series[self.framework_base]
427
self.native_arch = self._get_native_arch(system_arch, self.target_arch)
428
if chroots_dir is None:
429
chroots_dir = os.getcwd()
430
self.chroots_dir = chroots_dir
431
if "SUDO_USER" in os.environ:
432
self.user = os.environ["SUDO_USER"]
433
elif "PKEXEC_UID" in os.environ:
434
self.user = pwd.getpwuid(int(os.environ["PKEXEC_UID"])).pw_name
436
self.user = pwd.getpwuid(os.getuid()).pw_name
437
self.dpkg_architecture = self._dpkg_architecture()
439
def _get_native_arch(self, system_arch, target_arch):
440
"""Determine the proper native architecture for a chroot.
442
Some combinations of system and target architecture do not require
443
cross-building, so in these cases we just create a chroot suitable
446
if (system_arch, target_arch) in (
448
# This will only work if the system is running a 64-bit
449
# kernel; but there's no alternative since no i386-to-amd64
450
# cross-compiler is available in the Ubuntu archive.
457
def _dpkg_architecture(self):
458
dpkg_architecture = {}
459
command = ["dpkg-architecture", "-a%s" % self.target_arch]
460
env = dict(os.environ)
462
# Force dpkg-architecture to recalculate everything rather than
463
# picking up values from the environment, which will be present when
464
# running the test suite under dpkg-buildpackage.
465
for key in list(env):
466
if key.startswith("DEB_BUILD_") or key.startswith("DEB_HOST_"):
468
lines = subprocess.check_output(
469
command, env=env, universal_newlines=True).splitlines()
472
key, value = line.split("=", 1)
475
dpkg_architecture[key] = value
476
if self.native_arch == self.target_arch:
477
# We may have overridden the native architecture (see
478
# _get_native_arch above), so we need to force DEB_BUILD_* to
480
for key in list(dpkg_architecture):
481
if key.startswith("DEB_HOST_"):
482
new_key = "DEB_BUILD_" + key[len("DEB_HOST_"):]
483
dpkg_architecture[new_key] = dpkg_architecture[key]
484
return dpkg_architecture
486
def _generate_daemon_policy(self, mount):
487
daemon_policy = "%s/usr/sbin/policy-rc.d" % mount
488
with open(daemon_policy, "w") as policy:
489
policy.write(self.DAEMON_POLICY)
492
def _generate_apt_proxy_file(self, mount, proxy):
493
apt_conf_d = os.path.join(mount, "etc", "apt", "apt.conf.d")
494
if not os.path.exists(apt_conf_d):
495
os.makedirs(apt_conf_d)
496
apt_conf_f = os.path.join(apt_conf_d, "99-click-chroot-proxy")
498
with open(apt_conf_f, "w") as f:
500
// proxy settings copied by click chroot
509
def _generate_finish_script(self, mount, build_pkgs):
510
finish_script = "%s/finish.sh" % mount
511
with open(finish_script, 'w') as finish:
512
finish.write(dedent("""\
515
# Configure target arch
516
dpkg --add-architecture {target_arch}
517
# Reload package lists
518
apt-get update || true
519
# Pull down signature requirements
520
apt-get -y --force-yes install gnupg ubuntu-keyring
521
""").format(target_arch=self.target_arch))
522
if self.series == "vivid":
523
finish.write(dedent("""\
524
apt-get -y --force-yes install software-properties-common
525
add-apt-repository -y ppa:{ppa}
527
> /etc/apt/preferences.d/stable-phone-overlay.pref
529
"Pin: release o=LP-PPA-{pin_ppa}" \
530
>> /etc/apt/preferences.d/stable-phone-overlay.pref
531
echo "Pin-Priority: 1001" \
532
>> /etc/apt/preferences.d/stable-phone-overlay.pref
533
""").format(ppa=overlay_ppa,
534
pin_ppa=re.sub('/', '-', overlay_ppa)))
535
finish.write(dedent("""\
536
# Reload package lists
537
apt-get update || true
538
# Disable debconf questions
539
# so that automated builds won't prompt
540
echo set debconf/frontend Noninteractive | debconf-communicate
541
echo set debconf/priority critical | debconf-communicate
542
apt-get -y --force-yes dist-upgrade
543
# Install basic build tool set to match buildd
544
apt-get -y --force-yes install {build_pkgs}
545
# Make sure network is initialized
546
systemctl enable systemd-networkd.service
547
# Make sure dpkg variables are correct
548
for i in $(dpkg-architecture -a {target_arch} 2>/dev/null); \
549
do echo "export $i" >> /etc/profile.d/clickvars.sh ; \
554
[[ -e /sbin/initctl_tmp ]] && \
555
mv /sbin/initctl_tmp /sbin/initctl
556
rm /usr/sbin/policy-rc.d
557
""").format(build_pkgs=' '.join(build_pkgs),
558
target_arch=self.target_arch))
561
def _debootstrap(self, components, mount, archive_mirror, ports_mirror):
562
if self.native_arch in primary_arches:
563
mirror = archive_mirror
564
subprocess.check_call([
566
"--arch", self.native_arch,
568
"--components=%s" % ','.join(components),
574
mirror = ports_mirror
575
subprocess.check_call([
577
"--arch", self.native_arch,
580
"--components=%s" % ','.join(components),
587
def framework_base(self):
588
if self.framework in framework_base:
589
return framework_base[self.framework]
591
return self.framework
595
return "%s-%s-%s" % (self.framework_base,
599
def _make_executable(self, path):
600
mode = stat.S_IMODE(os.stat(path).st_mode)
601
os.chmod(path, mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
603
def _make_cross_package(self, prefix):
604
if self.native_arch == self.target_arch:
607
target_tuple = self.dpkg_architecture["DEB_HOST_GNU_TYPE"]
608
return "%s-%s" % (prefix, target_tuple)
610
def _is_crossbuild(self):
611
return (self.native_arch != self.target_arch)
614
components = ["main", "restricted", "universe", "multiverse"]
615
mount = "%s/%s" % (os.getcwd(), rootfs)
617
if not proxy and "http_proxy" in os.environ:
618
proxy = os.environ["http_proxy"]
620
proxy = subprocess.check_output(
621
'unset x; eval "$(apt-config shell x Acquire::HTTP::Proxy)"; \
623
shell=True, universal_newlines=True).strip()
625
# sort alphabetically
631
"libc-dev:%s" % self.target_arch,
632
# build pkg names dynamically
633
self._make_cross_package("g++"),
634
self._make_cross_package("pkg-config"),
636
for package in extra_packages.get(self.framework_base, []):
637
package = package.format(TARGET=self.target_arch)
638
build_pkgs.append(package)
639
if not self._is_crossbuild():
640
for package in runtime_packages:
641
package = package.format(TARGET=self.target_arch)
642
build_pkgs.append(package)
643
if not os.path.exists(mount):
650
" directory already exists")
652
country_code = get_geoip_country_code_prefix()
653
archive_mirror = "http://%sarchive.ubuntu.com/ubuntu" % country_code
654
ports_mirror = "http://%sports.ubuntu.com/ubuntu-ports" % country_code
655
# this doesn't work because we are running this under sudo
656
if 'DEBOOTSTRAP_MIRROR' in os.environ:
657
archive_mirror = os.environ['DEBOOTSTRAP_MIRROR']
658
self._debootstrap(components, mount, archive_mirror, ports_mirror)
659
# in case of creating foreign arch image we need to add the emulator
660
if self.native_arch not in primary_arches:
661
if os.path.isfile(qemu_arm_static):
662
shutil.copy2(qemu_arm_static, "%s/usr/bin/" % mount)
665
"The %s is not present, please install\
666
qemu-user-static package"
670
ret_code = subprocess.call(["/usr/sbin/chroot",
672
"/debootstrap/debootstrap",
676
"Second stage of the debootstrapping failed." +
679
sources = generate_sources(self.series, self.native_arch,
681
archive_mirror, ports_mirror,
682
' '.join(components))
683
with open("%s/etc/apt/sources.list" % mount, "w") as sources_list:
685
print(line, file=sources_list)
686
daemon_policy = self._generate_daemon_policy(mount)
687
self._make_executable(daemon_policy)
688
initctl = "%s/sbin/initctl" % mount
689
if os.path.exists(initctl):
690
shutil.copyfile(initctl, initctl + "_tmp")
692
os.symlink("%s/bin/true" % mount, initctl)
693
self._generate_apt_proxy_file(mount, proxy)
694
finish_script = self._generate_finish_script(mount, build_pkgs)
695
self._make_executable(finish_script)
696
ret_code = subprocess.call(["/usr/sbin/chroot", mount, "/finish.sh"])
699
"Debootstrapping the chroot failed." +
702
metadata = metadata_template.format(
703
ARCH=lxd_arch[self.native_arch],
704
CREATETIME=int(time.time()),
705
FULLNAME=self.full_name,
708
with open("metadata.yaml", "w") as metadata_file:
709
print(metadata, file=metadata_file)
710
# make sure /tmp is not cleaned when the chroot boots
711
with open("%s/etc/tmpfiles.d/tmp.conf" % mount,
712
"w") as notmpclean_file:
713
print("d /tmp 1777 root root -", file=notmpclean_file)
715
with open("%s/etc/network/interfaces" % mount, "w") as network_file:
716
print(network_settings, file=network_file)
717
# setup sshd for running and debugging
718
if not self._is_crossbuild():
719
with open("%s/etc/ssh/sshd_config" % mount, "a") as sshd_conf_file:
720
print("\nAuthorizedKeysFile /etc/ssh/authorized_keys.d/%u",
722
if not os.path.exists(template_directory):
723
os.makedirs(template_directory)
724
print("%s/%s" % (template_directory, hostname_tpl_filename))
725
with open("%s/%s" % (template_directory,
726
hostname_tpl_filename),
727
"w") as hostname_tpl_file:
728
print(hostname_tpl, file=hostname_tpl_file)
729
with open("%s/%s" % (template_directory,
731
"w") as hosts_tpl_file:
732
print(hosts_tpl, file=hosts_tpl_file)
734
with open("%s/%s" % (template_directory,
735
upstart_override_tpl_filename),
736
"w") as upstart_override_tpl_file:
737
print(upstart_override_tpl, file=upstart_override_tpl_file)
738
print("Packaging the LXD image.....")
739
rootfs_parts = os.listdir(rootfs)
740
ret = subprocess.call(["tar",
745
% self.full_name] + rootfs_parts)
746
ret = subprocess.call(["tar",
748
"%s-lxd.tar.xz" % self.full_name,
751
print("Clean up the build artifacts.")
752
shutil.rmtree(rootfs)
753
shutil.rmtree("templates")
754
os.remove("metadata.yaml")
755
print("Create the download.json")
756
add_image_to_download(self.framework,
763
def check_valid_arch(value):
764
valid_arch_set = ["i386", "amd64", "armhf"]
765
if value not in valid_arch_set:
766
raise argparse.ArgumentTypeError("%s is not a valid arch type" % value)
770
def check_valid_framework(value):
771
if value not in framework_series:
772
raise argparse.ArgumentTypeError("%s is not a valid arch type" % value)
776
def check_valid_type(value):
777
ota_re = '^ota\d+$|^dev$'
778
if not re.search(ota_re, value):
779
raise argparse.ArgumentTypeError("%s is not a valid image type"
785
if os.geteuid() != 0:
786
exit("You need to have" +
790
" to run this tool.\nPlease try again, using 'sudo'")
792
parser = argparse.ArgumentParser(description="Ubuntu SDK target builder")
793
parser.add_argument('-a',
794
'--target_architecture',
797
type=check_valid_arch)
798
parser.add_argument('-b',
799
'--base_architecture',
802
type=check_valid_arch)
803
parser.add_argument('-f',
806
default="ubuntu-sdk-15.04",
807
type=check_valid_framework)
808
parser.add_argument('-t',
812
type=check_valid_type)
813
options = parser.parse_args()
814
if os.isatty(sys.stdout.fileno()):
821
options.target_architecture +
825
options.base_architecture + bcolors.ENDC +
826
" base arch with " + bcolors.OKGREEN +
827
options.framework + bcolors.ENDC +
831
click = ClickChroot(options.base_architecture,
832
options.target_architecture,
837
sys.exit(click.create())
added
removed
share/qtcreator/ubuntu/scripts/usdk-target-build
