~ubuntu-archive/ubuntu-archive-scripts/trunk

#! /usr/bin/python3

# Based on lib/lp/archivepublisher/scripts/generate_extra_overrides.py from

# Launchpad, which is:

#

# Copyright 2011-2012 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

import copy

from functools import partial

import glob

import logging

from optparse import OptionParser

import os

import pickle

import shutil

import sys

import traceback

from germinate.archive import TagFile

from germinate.germinator import Germinator

from germinate.log import GerminateFormatter

from germinate.seeds import SeedError, SeedStructure

from launchpadlib.launchpad import Launchpad

import lputils

class AtomicFile:

    """Facilitate atomic writing of files."""

    def __init__(self, filename):

        self.filename = filename

        self.fd = open("%s.new" % self.filename, "w")

    def __enter__(self):

        return self.fd

    def __exit__(self, exc_type, exc_value, exc_tb):

        self.fd.close()

        if exc_type is None:

            os.rename("%s.new" % self.filename, self.filename)

class BufferHandler(logging.Handler):

    """A log handler which stores records for emission by another logger."""

    def __init__(self):

        super(BufferHandler, self).__init__()

        self.records = []

    def emit(self, record):

        # Record arguments may not be available at the other end.

        record_copy = copy.copy(record)

        record_copy.msg = record.getMessage()

        record_copy.args = None

        self.records.append(pickle.dumps(record_copy, -1))

class ExtraGerminate:

    def __init__(self, args):

        self.germinate_logger = None

        self.raw_args = args

    def processOptions(self):

        """Handle command-line options."""

        parser = OptionParser()

        parser.add_option(

            "-l", "--launchpad", dest="launchpad_instance",

            default="production")

        parser.add_option(

            "-d", "--distribution", dest="distribution", metavar="DISTRO",

            default="ubuntu",

            help="look in distribution DISTRO")

        parser.add_option("-s", "--suite", help="process this suite")

        parser.add_option(

            "-o", "--output", default="germinate-for-proposed",

            help="output to this directory")

        parser.add_option(

            "--archiveroot",

            default="file://%s" % os.path.expanduser("~/mirror/ubuntu/"),

            help="distribution archive URL")

        self.options, self.args = parser.parse_args(self.raw_args)

        self.options.launchpad = Launchpad.login_anonymously(

            'germinate-for-proposed', self.options.launchpad_instance)

        lputils.setup_location(self.options, default_pocket="Proposed")

        self.germinateroot = "%s.new" % self.options.output

    def setUpDirs(self):

        """Create output directories if they did not already exist."""

        if os.path.exists(self.germinateroot):

            shutil.rmtree(self.germinateroot)

        logging.debug("Creating germinate root %s." % self.germinateroot)

        os.makedirs(self.germinateroot)

    def addLogHandler(self):

        """Send germinate's log output to a separate file."""

        if self.germinate_logger is not None:

            return

        self.germinate_logger = logging.getLogger("germinate")

        self.germinate_logger.setLevel(logging.INFO)

        self.log_file = os.path.join(self.germinateroot, "germinate.output")

        self.log_handler = logging.FileHandler(self.log_file, mode="w")

        self.log_handler.setFormatter(GerminateFormatter())

        self.germinate_logger.addHandler(self.log_handler)

        self.germinate_logger.propagate = False

    def setUp(self):

        """Process options, and set up internal state."""

        self.processOptions()

        self.setUpDirs()

        self.addLogHandler()

    def getComponents(self, series):

        """Get the list of components to process for a given distroseries."""

        return ["main", "restricted", "universe", "multiverse"]

    def makeSeedStructures(self, suite, flavours, seed_bases=None):

        series_name = suite.split("-")[0]

        structures = {}

        for flavour in flavours:

            try:

                structure = SeedStructure(

                    "%s.%s" % (flavour, series_name), seed_bases=seed_bases)

                if len(structure):

                    structures[flavour] = structure

                else:

                    logging.warning(

                        "Skipping empty seed structure for %s.%s",

                        flavour, series_name)

            except SeedError as e:

                logging.warning(

                    "Failed to fetch seeds for %s.%s: %s",

                    flavour, series_name, e)

        return structures

    def logGerminateProgress(self, *args):

        """Log a "progress" entry to the germinate log file.

        Germinate logs quite a bit of detailed information.  To make it

        easier to see the structure of its operation, GerminateFormatter

        allows tagging some log entries as "progress" entries, which are

        printed without a prefix.

        """

        self.germinate_logger.info(*args, extra={"progress": True})

    def composeOutputPath(self, flavour, series_name, arch, base):

        return os.path.join(

            self.germinateroot,

            "%s_%s_%s_%s" % (base, flavour, series_name, arch))

    def recordOutput(self, path, seed_outputs):

        if seed_outputs is not None:

            seed_outputs.add(os.path.basename(path))

    def writeGerminateOutput(self, germinator, structure, flavour,

                             suite, arch, seed_outputs=None):

        """Write dependency-expanded output files.

        These files are a reduced subset of those written by the germinate

        command-line program.

        """

        series_name = suite.split("-")[0]

        path = partial(self.composeOutputPath, flavour, series_name, arch)

        # The structure file makes it possible to figure out how the other

        # output files relate to each other.

        structure.write(path("structure"))

        self.recordOutput(path("structure"), seed_outputs)

        # "all" and "all.sources" list the full set of binary and source

        # packages respectively for a given flavour/suite/architecture

        # combination.

        germinator.write_all_list(structure, path("all"))

        self.recordOutput(path("all"), seed_outputs)

        germinator.write_all_source_list(structure, path("all.sources"))

        self.recordOutput(path("all.sources"), seed_outputs)

        # Write the dependency-expanded output for each seed.  Several of

        # these are used by archive administration tools, and others are

        # useful for debugging, so it's best to just write them all.

        for seedname in structure.names:

            germinator.write_full_list(structure, path(seedname), seedname)

            self.recordOutput(path(seedname), seed_outputs)

    def germinateArchFlavour(self, germinator, suite, arch, flavour,

                             structure, primary_flavour, seed_outputs=None):

        """Germinate seeds on a single flavour for a single architecture."""

        # Expand dependencies.

        germinator.plant_seeds(structure)

        germinator.grow(structure)

        germinator.add_extras(structure)

        self.writeGerminateOutput(

            germinator, structure, flavour, suite, arch,

            seed_outputs=seed_outputs)

    def germinateArch(self, suite, components, arch, flavours, structures):

        """Germinate seeds on all flavours for a single architecture."""

        # Buffer log output for each architecture so that it appears

        # sequential.

        self.germinate_logger.removeHandler(self.log_handler)

        log_handler = BufferHandler()

        self.germinate_logger.addHandler(log_handler)

        germinator = Germinator(arch)

        # Compute ogre model.

        if "-" in suite:

            series, pocket = suite.split("-")

            suites = [series]

            if pocket in ("updates", "security", "proposed"):

                suites.append("%s-security" % series)

            if pocket in ("updates", "proposed"):

                suites.append("%s-updates" % series)

            if pocket == "proposed":

                suites.append("%s-proposed" % series)

        else:

            suites = [suite]

        # Read archive metadata.

        archive = TagFile(

            suites, components, arch, self.options.archiveroot, cleanup=True)

        germinator.parse_archive(archive)

        seed_outputs = set()

        for flavour in flavours:

            logging.info(

                "Germinating for %s/%s/%s", flavour, suite, arch)

            # Add this to the germinate log as well so that that can be

            # debugged more easily.  Log a separator line first.

            self.logGerminateProgress("")

            self.logGerminateProgress(

                "Germinating for %s/%s/%s", flavour, suite, arch)

            self.germinateArchFlavour(

                germinator, suite, arch, flavour,

                structures[flavour], flavour == flavours[0],

                seed_outputs=seed_outputs)

        return log_handler.records, seed_outputs

    def germinateArchChild(self, close_in_child, wfd, *args):

        """Helper method to call germinateArch in a forked child process."""

        try:

            for fd in close_in_child:

                os.close(fd)

            with os.fdopen(wfd, "wb") as writer:

                pickle.dump(self.germinateArch(*args), writer, -1)

            return 0

        except:

            traceback.print_exc()

            pickle.dump(([], set()), writer, -1)

            return 1

    def removeStaleOutputs(self, suite, seed_outputs):

        """Remove stale outputs for a suite.

        Any per-seed outputs not in seed_outputs are considered stale.

        """

        series_name = suite.split("-")[0]

        all_outputs = glob.glob(

            os.path.join(self.germinateroot, "*_*_%s_*" % series_name))

        for output in all_outputs:

            if os.path.basename(output) not in seed_outputs:

                os.remove(output)

    def generateExtraOverrides(self, suite, components, architectures,

                               flavours, seed_bases=None):

        structures = self.makeSeedStructures(

            suite, flavours, seed_bases=seed_bases)

        if structures:

            procs = []

            close_in_child = []

            for arch in architectures:

                rfd, wfd = os.pipe()

                close_in_child.append(rfd)

                pid = os.fork()

                if pid == 0:  # child

                    os._exit(self.germinateArchChild(

                        close_in_child, wfd,

                        suite, components, arch, flavours, structures))

                else:  # parent

                    os.close(wfd)

                    reader = os.fdopen(rfd, "rb")

                    procs.append((pid, reader))

            seed_outputs = set()

            for pid, reader in procs:

                log_records, arch_seed_outputs = pickle.load(reader)

                for log_record in log_records:

                    self.germinate_logger.handle(pickle.loads(log_record))

                seed_outputs |= arch_seed_outputs

                reader.close()

                os.waitpid(pid, 0)

            self.removeStaleOutputs(suite, seed_outputs)

    def process(self, seed_bases=None):

        """Do the bulk of the work."""

        self.setUp()

        components = self.getComponents(self.options.series)

        architectures = [

            a.architecture_tag for a in self.options.architectures]

        self.generateExtraOverrides(

            self.options.suite, components, architectures, self.args,

            seed_bases=seed_bases)

        if os.path.exists("%s.old" % self.options.output):

            shutil.rmtree("%s.old" % self.options.output)

        if os.path.exists(self.options.output):

            os.rename(self.options.output, "%s.old" % self.options.output)

        os.rename(self.germinateroot, self.options.output)

        if os.path.exists("%s.old" % self.options.output):

            shutil.rmtree("%s.old" % self.options.output)

if __name__ == "__main__":

    ExtraGerminate(sys.argv[1:]).process()