← Back to branch summary

~xubuntu-dev/ubuntu-cdimage/xubuntu-base

« back to all changes in this revision

Viewing changes to lib/cdimage/build.py

  • Committer: Colin Watson
  • Date: 2015-09-14 07:42:14 UTC
  • Revision ID: cjwatson@canonical.com-20150914074214-6utmkgmobnz5cjvh
Silence some noise from checksum tests.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/cdimage/build.py
23
23
import gzip
24
24
import os
25
25
import shutil
 
26
import socket
26
27
import stat
27
28
import subprocess
28
29
import sys
33
34
from cdimage.build_id import next_build_id
34
35
from cdimage.check_installable import check_installable
35
36
from cdimage.germinate import Germination
36
 
from cdimage.livefs import download_live_filesystems, live_output_directory
 
37
from cdimage.livefs import (
 
38
    LiveBuildsFailed,
 
39
    download_live_filesystems,
 
40
    live_output_directory,
 
41
    run_live_builds,
 
42
)
37
43
from cdimage.log import logger, reset_logging
38
44
from cdimage.mail import get_notify_addresses, send_mail
39
45
from cdimage.mirror import find_mirror, trigger_mirrors
40
46
from cdimage.semaphore import Semaphore
 
47
from cdimage.tracker import tracker_set_rebuild_status
41
48
from cdimage.tree import Publisher, Tree
 
49
from cdimage.config import Touch
42
50
 
43
51
 
44
52
@contextlib.contextmanager
46
54
    project = config.project
47
55
    if config["UBUNTU_DEFAULTS_LOCALE"] == "zh_CN":
48
56
        project = "ubuntu-chinese-edition"
 
57
    if config.distribution == "ubuntu":
 
58
        full_series = config.series
 
59
    else:
 
60
        full_series = "%s-%s" % (config.distribution, config.series)
49
61
    lock_path = os.path.join(
50
62
        config.root, "etc",
51
63
        ".lock-build-image-set-%s-%s-%s" % (
52
 
            project, config.series, config.image_type))
 
64
            project, full_series, config.image_type))
53
65
    try:
54
66
        subprocess.check_call(["lockfile", "-l", "7200", "-r", "0", lock_path])
55
67
    except subprocess.CalledProcessError:
77
89
            config["CDIMAGE_UNSUPPORTED"] = "1"
78
90
    elif project in (
79
91
        "kubuntu-active",
 
92
        "kubuntu-plasma5",
80
93
        "ubuntustudio",
81
94
        "mythbuntu",
82
95
        "lubuntu",
83
96
        "ubuntukylin",
84
97
        "ubuntu-gnome",
 
98
        "ubuntu-mate",
85
99
        "ubuntu-moblin-remix",
86
100
        "ubuntu-mid",
87
101
    ):
92
106
 
93
107
 
94
108
def open_log(config):
95
 
    if config["DEBUG"]:
 
109
    if config["DEBUG"] or config["CDIMAGE_NOLOG"]:
96
110
        return None
97
111
 
98
112
    project = config.project
99
113
    if config["UBUNTU_DEFAULTS_LOCALE"] == "zh_CN":
100
114
        project = "ubuntu-chinese-edition"
101
115
    log_path = os.path.join(
102
 
        config.root, "log", project, config.series,
 
116
        config.root, "log", project, config.full_series,
103
117
        "%s-%s.log" % (config.image_type, config["CDIMAGE_DATE"]))
104
118
    osextras.ensuredir(os.path.dirname(log_path))
105
119
    log = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666)
118
132
 
119
133
def log_marker(message):
120
134
    logger.info("===== %s =====" % message)
121
 
    logger.info(time.strftime("%a %b %e %H:%M:%S %Z %Y", time.gmtime()))
 
135
    logger.info(time.strftime("%a %b %e %H:%M:%S UTC %Y", time.gmtime()))
 
136
 
 
137
 
 
138
def want_live_builds(options):
 
139
    return options is not None and getattr(options, "live", False)
 
140
 
 
141
 
 
142
def _anonftpsync_config_path(config):
 
143
    if config["ANONFTPSYNC_CONF"]:
 
144
        return config["ANONFTPSYNC_CONF"]
 
145
    paths = [
 
146
        os.path.join(config.root, "production", "anonftpsync"),
 
147
        os.path.join(config.root, "etc", "anonftpsync"),
 
148
    ]
 
149
    for path in paths:
 
150
        if os.path.exists(path):
 
151
            return path
 
152
    else:
 
153
        return None
 
154
 
 
155
 
 
156
def _anonftpsync_options(config):
 
157
    env = {}
 
158
    path = _anonftpsync_config_path(config)
 
159
    if path:
 
160
        whitelisted_keys = [
 
161
            "RSYNC_EXCLUDE",
 
162
            "RSYNC_ICONV",
 
163
            "RSYNC_PASSWORD",
 
164
            "RSYNC_PROXY",
 
165
            "RSYNC_RSH",
 
166
            "RSYNC_SRC",
 
167
        ]
 
168
        for key, value in osextras.read_shell_config(path, whitelisted_keys):
 
169
            if key.startswith("RSYNC_"):
 
170
                env[key] = value
 
171
    if "RSYNC_SRC" not in env:
 
172
        raise Exception(
 
173
            "RSYNC_SRC not configured!  Edit %s or %s and try again." % (
 
174
                os.path.join(config.root, "production", "anonftpsync"),
 
175
                os.path.join(config.root, "etc", "anonftpsync")))
 
176
    return env
 
177
 
 
178
 
 
179
def anonftpsync(config):
 
180
    env = dict(os.environ)
 
181
    for key, value in _anonftpsync_options(config).items():
 
182
        env[key] = value
 
183
    target = os.path.join(config.root, "ftp")
 
184
    fqdn = socket.getfqdn()
 
185
    lock_base = "Archive-Update-in-Progress-%s" % fqdn
 
186
    lock = os.path.join(target, lock_base)
 
187
    if subprocess.call(
 
188
            ["lockfile", "-!", "-l", "43200", "-r", "0", lock]) == 0:
 
189
        raise Exception(
 
190
            "%s is unable to start rsync; lock file exists." % fqdn)
 
191
    try:
 
192
        log_path = os.path.join(config.root, "log", "rsync.log")
 
193
        osextras.ensuredir(os.path.dirname(log_path))
 
194
        with open(log_path, "w") as log:
 
195
            command_base = [
 
196
                "rsync", "--recursive", "--links", "--hard-links", "--times",
 
197
                "--verbose", "--stats", "--chmod=Dg+s,g+rwX",
 
198
                "--exclude", lock_base,
 
199
                "--exclude", "project/trace/%s" % fqdn,
 
200
            ]
 
201
            exclude = env.get("RSYNC_EXCLUDE", "").split()
 
202
            source_target = ["%s/" % env["RSYNC_SRC"], "%s/" % target]
 
203
 
 
204
            subprocess.call(
 
205
                command_base + [
 
206
                    "--exclude", "Packages*", "--exclude", "Sources*",
 
207
                    "--exclude", "Release*", "--exclude", "InRelease",
 
208
                ] + exclude + source_target,
 
209
                stdout=log, stderr=subprocess.STDOUT, env=env)
 
210
 
 
211
            # Second pass to update metadata and clean up old files.
 
212
            subprocess.call(
 
213
                command_base + [
 
214
                    "--delay-updates", "--delete", "--delete-after",
 
215
                ] + exclude + source_target,
 
216
                stdout=log, stderr=subprocess.STDOUT, env=env)
 
217
 
 
218
        # Delete dangling symlinks.
 
219
        for dirpath, _, filenames in os.walk(target):
 
220
            for filename in filenames:
 
221
                path = os.path.join(dirpath, filename)
 
222
                if os.path.islink(path) and not os.path.exists(path):
 
223
                    os.unlink(path)
 
224
 
 
225
        trace_dir = os.path.join(target, "project", "trace")
 
226
        osextras.ensuredir(trace_dir)
 
227
        with open(os.path.join(trace_dir, fqdn), "w") as trace:
 
228
            subprocess.check_call(["date", "-u"], stdout=trace)
 
229
 
 
230
        # Note: if you don't have savelog, use any other log rotation
 
231
        # facility, or comment this out, the log will simply be overwritten
 
232
        # each time.
 
233
        with open("/dev/null", "w") as devnull:
 
234
            subprocess.call(
 
235
                ["savelog", log_path],
 
236
                stdout=devnull, stderr=subprocess.STDOUT)
 
237
    finally:
 
238
        osextras.unlink_force(lock)
122
239
 
123
240
 
124
241
def sync_local_mirror(config, semaphore_state):
137
254
            logger.error("Couldn't acquire archive sync lock!")
138
255
            raise
139
256
        try:
140
 
            subprocess.check_call(["anonftpsync"])
 
257
            anonftpsync(config)
141
258
        finally:
142
259
            osextras.unlink_force(sync_lock)
143
260
    else:
286
403
            else:
287
404
                osextras.unlink_force("%s.%s" % (scratch_prefix, ext))
288
405
    else:
 
406
        log_marker("Downloading live filesystem images")
289
407
        download_live_filesystems(config)
290
408
        scratch = live_output_directory(config)
291
409
        for entry in os.listdir(scratch):
292
410
            if "." in entry:
293
411
                os.rename(
294
412
                    os.path.join(scratch, entry),
295
 
                    os.path.join(
296
 
                        scratch, "%s-desktop-%s" % (series.name, entry)))
 
413
                    os.path.join(scratch, "%s-desktop-%s" % (series, entry)))
297
414
        pi_makelist = os.path.join(
298
415
            config.root, "debian-cd", "tools", "pi-makelist")
299
416
        for entry in os.listdir(scratch):
305
422
                        [pi_makelist, entry_path], stdout=list_file)
306
423
 
307
424
 
 
425
def add_android_support(config, arch, output_dir):
 
426
    """Copy Android support files to an Ubuntu Touch image.
 
427
    """
 
428
    live_scratch_dir = os.path.join(
 
429
        config.root, "scratch", config.project, config.full_series,
 
430
        config.image_type, "live")
 
431
 
 
432
    # copy recovery, boot and system imgs in place
 
433
    for target in Touch.list_targets_by_ubuntu_arch(arch):
 
434
        boot_img_src = "boot-%s+%s.img" % (target.ubuntu_arch, target.subarch)
 
435
        boot_img = "%s-preinstalled-boot-%s+%s.img" % (
 
436
            config.series, arch, target.subarch)
 
437
        system_img_src = "system-%s+%s.img" % (
 
438
            target.android_arch, target.subarch)
 
439
        system_img = "%s-preinstalled-system-%s+%s.img" % (
 
440
            config.series, target.android_arch, target.subarch)
 
441
        recovery_img_src = "recovery-%s+%s.img" % (
 
442
            target.android_arch, target.subarch)
 
443
        recovery_img = "%s-preinstalled-recovery-%s+%s.img" % (
 
444
            config.series, target.android_arch, target.subarch)
 
445
 
 
446
        shutil.copy2(
 
447
            os.path.join(live_scratch_dir, boot_img_src),
 
448
            os.path.join(output_dir, boot_img))
 
449
        shutil.copy2(
 
450
            os.path.join(live_scratch_dir, system_img_src),
 
451
            os.path.join(output_dir, system_img))
 
452
        shutil.copy2(
 
453
            os.path.join(live_scratch_dir, recovery_img_src),
 
454
            os.path.join(output_dir, recovery_img))
 
455
 
 
456
 
 
457
def build_livecd_base(config):
 
458
    log_marker("Downloading live filesystem images")
 
459
    download_live_filesystems(config)
 
460
 
 
461
    if (config.project in ("ubuntu-core", "ubuntu-touch") or
 
462
        (config.project == "ubuntu-desktop-next" and
 
463
         config.subproject == "system-image")):
 
464
        log_marker("Copying images to debian-cd output directory")
 
465
        scratch_dir = os.path.join(
 
466
            config.root, "scratch", config.project, config.full_series,
 
467
            config.image_type)
 
468
        live_dir = os.path.join(scratch_dir, "live")
 
469
        for arch in config.arches:
 
470
            live_prefix = os.path.join(live_dir, arch)
 
471
            rootfs = "%s.rootfs.tar.gz" % live_prefix
 
472
            if os.path.exists(rootfs):
 
473
                output_dir = os.path.join(scratch_dir, "debian-cd", arch)
 
474
                osextras.ensuredir(output_dir)
 
475
                if config.project == "ubuntu-core":
 
476
                    if config.image_type == "daily-preinstalled":
 
477
                        output_prefix = os.path.join(
 
478
                            output_dir,
 
479
                            "%s-preinstalled-core-%s" % (config.series, arch))
 
480
                    else:
 
481
                        output_prefix = os.path.join(
 
482
                            output_dir, "%s-core-%s" % (config.series, arch))
 
483
                elif config.project == "ubuntu-touch":
 
484
                    output_prefix = os.path.join(
 
485
                        output_dir,
 
486
                        "%s-preinstalled-touch-%s" % (config.series, arch))
 
487
                elif config.project == "ubuntu-desktop-next":
 
488
                    if config.image_type == "daily-preinstalled":
 
489
                        output_prefix = os.path.join(
 
490
                            output_dir,
 
491
                            "%s-preinstalled-desktop-next-%s" %
 
492
                            (config.series, arch))
 
493
                    else:
 
494
                        output_prefix = os.path.join(
 
495
                            output_dir, "%s-desktop-next-%s" %
 
496
                            (config.series, arch))
 
497
                shutil.copy2(rootfs, "%s.raw" % output_prefix)
 
498
                with open("%s.type" % output_prefix, "w") as f:
 
499
                    print("tar archive", file=f)
 
500
                shutil.copy2(
 
501
                    "%s.manifest" % live_prefix, "%s.manifest" % output_prefix)
 
502
                if config.project == "ubuntu-touch":
 
503
                    osextras.link_force(
 
504
                        "%s.raw" % output_prefix, "%s.tar.gz" % output_prefix)
 
505
                    add_android_support(config, arch, output_dir)
 
506
                    custom = "%s.custom.tar.gz" % live_prefix
 
507
                    if os.path.exists(custom):
 
508
                        shutil.copy2(
 
509
                            custom, "%s.custom.tar.gz" % output_prefix)
 
510
                if config.project in ("ubuntu-core", "ubuntu-desktop-next"):
 
511
                    for dev in ("azure.device", "device"):
 
512
                        device = "%s.%s.tar.gz" % (live_prefix, dev)
 
513
                        if os.path.exists(device):
 
514
                            shutil.copy2(
 
515
                                device, "%s.%s.tar.gz" % (output_prefix, dev))
 
516
 
 
517
 
308
518
def _debootstrap_script(config):
309
519
    if config["DIST"] <= "gutsy":
310
520
        return "usr/lib/debootstrap/scripts/%s" % config.series
313
523
 
314
524
 
315
525
def extract_debootstrap(config):
316
 
    series = config["DIST"]
317
526
    output_dir = os.path.join(
318
 
        config.root, "scratch", config.project, series.name, config.image_type,
319
 
        "debootstrap")
 
527
        config.root, "scratch", config.project, config.full_series,
 
528
        config.image_type, "debootstrap")
320
529
 
321
530
    osextras.ensuredir(output_dir)
322
531
 
325
534
        mirror = find_mirror(config, arch)
326
535
        # TODO: This might be more sensible with python-debian or python-apt.
327
536
        packages_path = os.path.join(
328
 
            mirror, "dists", series.name, "main", "debian-installer",
 
537
            mirror, "dists", config.series, "main", "debian-installer",
329
538
            "binary-%s" % arch, "Packages.gz")
330
539
        with gzip.GzipFile(packages_path, "rb") as packages:
331
540
            grep_dctrl = subprocess.Popen(
378
587
def fix_permissions(config):
379
588
    """Kludge to work around permission-handling problems elsewhere."""
380
589
    scratch_dir = os.path.join(
381
 
        config.root, "scratch", config.project, config.series,
 
590
        config.root, "scratch", config.project, config.full_series,
382
591
        config.image_type)
383
592
    if not os.path.isdir(scratch_dir):
384
593
        return
413
622
 
414
623
 
415
624
def notify_failure(config, log_path):
416
 
    if config["DEBUG"]:
 
625
    if config["DEBUG"] or config["CDIMAGE_NOLOG"]:
417
626
        return
418
627
 
419
628
    project = config.project
420
629
    if config["UBUNTU_DEFAULTS_LOCALE"] == "zh_CN":
421
630
        project = "ubuntu-chinese-edition"
422
 
    series = config.series
 
631
    series = config.full_series
423
632
    image_type = config.image_type
424
633
    date = config["CDIMAGE_DATE"]
425
634
 
433
642
        else:
434
643
            body = open(log_path)
435
644
        send_mail(
436
 
            "CD image %s/%s/%s failed to build on %s" % (
 
645
            "CD image %s%s/%s/%s failed to build on %s" % (
 
646
                ("(built by %s) " % config["SUDO_USER"]
 
647
                 if config["SUDO_USER"] else ""),
437
648
                project, series, image_type, date),
438
649
            "build-image-set", recipients, body)
439
650
    finally:
441
652
            body.close()
442
653
 
443
654
 
444
 
def build_image_set_locked(config, semaphore_state):
 
655
def is_live_fs_only(config):
 
656
    live_fs_only = False
 
657
    if config.project in ("livecd-base", "ubuntu-core", "ubuntu-touch"):
 
658
        live_fs_only = True
 
659
    elif (config.project == "ubuntu-desktop-next" and
 
660
          config.subproject == "system-image"):
 
661
        live_fs_only = True
 
662
    elif config.subproject == "wubi":
 
663
        live_fs_only = True
 
664
    return live_fs_only
 
665
 
 
666
 
 
667
def build_image_set_locked(config, options, semaphore_state):
445
668
    image_type = config.image_type
446
669
    config["CDIMAGE_DATE"] = date = next_build_id(config, image_type)
447
670
    log_path = None
450
673
        configure_for_project(config)
451
674
        log_path = open_log(config)
452
675
 
453
 
        sync_local_mirror(config, semaphore_state)
454
 
 
455
 
        if config["LOCAL"]:
456
 
            log_marker("Updating archive of local packages")
457
 
            update_local_indices(config)
458
 
 
459
 
        build_britney(config)
460
 
 
461
 
        log_marker("Extracting debootstrap scripts")
462
 
        extract_debootstrap(config)
 
676
        if want_live_builds(options):
 
677
            log_marker("Building live filesystems")
 
678
            live_successful = run_live_builds(config)
 
679
            config.limit_arches(live_successful)
 
680
        else:
 
681
            tracker_set_rebuild_status(config, [0, 1], 2)
 
682
 
 
683
        if not is_live_fs_only(config):
 
684
            sync_local_mirror(config, semaphore_state)
 
685
 
 
686
            if config["LOCAL"]:
 
687
                log_marker("Updating archive of local packages")
 
688
                update_local_indices(config)
 
689
 
 
690
            build_britney(config)
 
691
 
 
692
            log_marker("Extracting debootstrap scripts")
 
693
            extract_debootstrap(config)
463
694
 
464
695
        if config["UBUNTU_DEFAULTS_LOCALE"]:
465
696
            build_ubuntu_defaults_locale(config)
 
697
        elif is_live_fs_only(config):
 
698
            build_livecd_base(config)
466
699
        else:
467
700
            if not config["CDIMAGE_PREINSTALLED"]:
468
701
                log_marker("Germinating")
474
707
                germinate_output.write_tasks()
475
708
 
476
709
                log_marker("Checking for other task changes")
477
 
                subprocess.check_call(["update-tasks", date, image_type])
 
710
                germinate_output.update_tasks(date)
478
711
 
479
712
            if (config["CDIMAGE_LIVE"] or config["CDIMAGE_SQUASHFS_BASE"] or
480
713
                    config["CDIMAGE_PREINSTALLED"]):
496
729
        if not config["DEBUG"] and not config["CDIMAGE_NOPUBLISH"]:
497
730
            log_marker("Publishing")
498
731
            tree = Tree.get_daily(config)
499
 
            Publisher.get_daily(tree, image_type).publish(date)
 
732
            publisher = Publisher.get_daily(tree, image_type)
 
733
            publisher.publish(date)
500
734
 
501
735
            log_marker("Purging old images")
502
 
            subprocess.check_call(["purge-old-images", image_type])
 
736
            publisher.purge()
503
737
 
504
738
            log_marker("Triggering mirrors")
505
739
            trigger_mirrors(config)
506
740
 
507
741
        log_marker("Finished")
508
742
        return True
509
 
    except Exception:
 
743
    except Exception as e:
510
744
        for line in traceback.format_exc().splitlines():
511
745
            logger.error(line)
512
746
        sys.stdout.flush()
513
747
        sys.stderr.flush()
514
 
        notify_failure(config, log_path)
 
748
        if not isinstance(e, LiveBuildsFailed):
 
749
            notify_failure(config, log_path)
515
750
        return False
516
751
 
517
752
 
518
 
def build_image_set(config):
 
753
def build_image_set(config, options):
519
754
    """Master entry point for building images."""
520
755
    semaphore = Semaphore(
521
756
        os.path.join(config.root, "etc", ".sem-build-image-set"))
522
757
    with lock_build_image_set(config), semaphore.held() as semaphore_state:
523
 
        return build_image_set_locked(config, semaphore_state)
 
758
        return build_image_set_locked(config, options, semaphore_state)