~ubuntu-archive/ubuntu-archive-scripts/trunk

#!/usr/bin/python3

# This script highlights packages that are being kept out of Debian

# testing by release critical bugs whose removal from the Ubuntu devel

# series would aid the migration of other packages out of proposed.

#

# The packages that are being kept out of Debian testing by release

# critical bugs can be found fairly easily by looking at the output

# from Debian's britney runs. We do this first (and call these

# "rc-gone packages").

#

# Such packages can inhibit the migration of other packages in two

# ways:

#

# 1) autopkgtest regressions

# 2) by becoming uninstallable

#

# The first is fairly easy to find: scan through Ubuntu's excuses.yaml

# and look for the rc-gone package in the list of autopkgtest

# regressions for any package.

#

# The second is a bit more mind-bending to detect. If the package is

# caught up in a transition, a rebuild will be attempted as a matter

# of course, and if this succeeds and passes its autopkgtests then

# removing the package will not aid proposed migration. So we look for

# rc-gone packages in proposed that are missing builds or failing

# autopkgtests. For such source packages, we see if any of their

# binary packages are reported as being made uninstallable by the

# migration of any source package in proposed. If so, removing the

# rc-gone package will help proposed migration.

import argparse

import os

import shlex

import subprocess

import attr

from jinja2 import Environment, FileSystemLoader

import yaml

env = Environment(

    loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),

    autoescape=True,

    extensions=['jinja2.ext.i18n'],

)

env.install_null_translations(True)

def parse_args():

    parser = argparse.ArgumentParser()

    parser.add_argument('--ubuntu-excuses', action='store')

    parser.add_argument('--ubuntu-update_output', action='store')

    parser.add_argument('--debian-excuses', action='store')

    parser.add_argument('--output', action='store')

    return parser.parse_args()

args = parse_args()

def run_output(*cmd, **extra):

    encoding = extra.pop('encoding', 'ascii')

    kw = dict(check=True, stdout=subprocess.PIPE)

    kw.update(extra)

    cp = subprocess.run(cmd, **kw)

    return cp.stdout.decode(encoding).strip()

all_arches = set()

def extract_bin_pkg_arch_to_blocked_src_pkgs(output_fp):

    # Extract a mapping from binary package name / architectures to

    # source package the migration of which would make that package

    # uninstallable (and for convenience, return all architectures

    # used as keys -- usually there will only be one or two).

    # We're looking for sequences of lines like this:

    # skipped: camlp5 (0, 3, 57)

    #     got: 13+0: a-1:a-0:a-0:i-6:p-0:s-6

    #      * s390x: hol-light, libaac-tactics-ocaml-dev, libcoq-ocaml-dev, libledit-ocaml-dev, ocaml-ulex08

    # (Britney tries to migrate batches of packages but it always

    # tries each package on its own as well).

    # For each potential migration, britney checks architectures in

    # sequence and stops when it finds one that regresses (or proceeds

    # with the migration if it doesn't find one). This means that we

    # can miss blocking packages here --- for example if migrating $src

    # would make say $binpkg/amd64 uninstallable, but britney happens

    # to check arm64 -- where $binpkg does not exist -- and there are

    # regressions there, we will never find out about the problem

    # $binpkg causes.  This isn't really too bad because clearly in

    # this case the migration of $src is blocked by other things that

    # need to be resolved, but it does mean that packages might appear

    # and disapper from the report depending on the order that britney

    # checks architectures in (which is not consistent from run to

    # run). C'est la vie.

    bin_pkg_arch_to_blocked_src_pkgs = {}

    srcpkg = None

    arch_prefix = "Arch order is: "

    for line in output_fp:

        line = line.strip()

        if line.startswith(arch_prefix):

            all_arches.update(line[len(arch_prefix):].split(', '))

        parts = line.split(None, 2)

        if len(parts) >= 2:

            if parts[0] in {"Trying", "trying"}:

                srcpkg = None

            if parts[0] == 'skipped:':

                srcpkg = None

                # If parts[2] is '(' then this line is about trying to

                # migrate a single package, which is what we are

                # looking for.

                if parts[2].startswith('('):

                    srcpkg = parts[1]

            if srcpkg is not None and srcpkg[0] != '-' and parts[0] == '*':

                # parts[1] is "${arch}:"

                # parts[2:] is a comma+space separated list of binary package names.

                arch = parts[1][:-1]

                for binpkg in parts[2].split(', '):

                    bin_pkg_arch_to_blocked_src_pkgs.setdefault(

                        (binpkg, arch), set()).add(srcpkg)

    return bin_pkg_arch_to_blocked_src_pkgs

def chdist_grep_dctrl_packages(arch, *args):

    return run_output(

        "chdist", "grep-dctrl-packages", "{}-{}".format(series, arch),

        "-nsPackage", *args, check=False).splitlines()

def chdist_grep_dctrl_sources(arch, *args):

    return run_output(

        "chdist", "grep-dctrl-sources", "{}-{}".format(series, arch),

        "-nsPackage", *args, check=False).splitlines()

@attr.s

class RCGone:

    source_package_name = attr.ib()

    bugs = attr.ib()

    block_by_regression = attr.ib(default=attr.Factory(set))

    block_by_uninstallability = attr.ib(default=attr.Factory(set))

    suites = attr.ib(default=attr.Factory(set))

    _rdeps_lines = attr.ib(default=None)

    _binpkgs = attr.ib(default=None)

    def binary_pkgs(self):

        if self._binpkgs is None:

            self._binpkgs = set()

            for arch in all_arches:

                arch_binpkgs = chdist_grep_dctrl_packages(

                    arch, "-wS", self.source_package_name)

                self._binpkgs.update({(binpkg, arch) for binpkg in arch_binpkgs})

        return self._binpkgs

    def reverse_depends(self):

        if self._rdeps_lines is None:

            # These are maps rdep -> binpkg -> arches

            reverse_recommends = {}

            reverse_depends = {}

            # This just maps rbdep -> binpkgs

            reverse_build_depends = {}

            for binpkg, arch in self.binary_pkgs():

                for rec in chdist_grep_dctrl_packages(

                        arch, "-wFRecommends", binpkg):

                    if (rec, arch) in self.binary_pkgs():

                        continue

                    reverse_recommends.setdefault(rec, {}).setdefault(binpkg, set()).add(arch)

                for dep in chdist_grep_dctrl_packages(

                        arch, "-wFDepends", binpkg):

                    if (dep, arch) in self.binary_pkgs():

                        continue

                    reverse_depends.setdefault(dep, {}).setdefault(binpkg, set()).add(arch)

                for bdeb in chdist_grep_dctrl_sources(

                    arch, "-wFBuild-Depends", binpkg, "--or", "-wFBuild-Depends-Indep", binpkg):

                    reverse_build_depends.setdefault(bdeb, set()).add(binpkg)

            self._rdeps_lines = []

            if reverse_depends:

                self._rdeps_lines.append("Reverse-Depends")

                for rdep in sorted(reverse_depends):

                    for binpkg in sorted(reverse_depends[rdep]):

                        if reverse_depends[rdep][binpkg] == all_arches:

                            arches = "all architectures"

                        else:

                            arches = ", ".join(reverse_depends[rdep][binpkg])

                        self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg, "on", arches]))

            if reverse_recommends:

                self._rdeps_lines.append("Reverse-Recommends")

                for rdep in sorted(reverse_recommends):

                    for binpkg in sorted(reverse_recommends[rdep]):

                        if reverse_recommends[rdep][binpkg] == all_arches:

                            arches = "all architectures"

                        else:

                            arches = ", ".join(reverse_recommends[rdep][binpkg])

                        self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg, "on", arches]))

            if reverse_build_depends:

                self._rdeps_lines.append("Reverse-Build-Depends")

                for rdep in sorted(reverse_build_depends):

                    for binpkg in sorted(reverse_build_depends[rdep]):

                        self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg]))

        return self._rdeps_lines

    @property

    def comment(self):

        comment = "removed from testing (Debian bug"

        if len(self.bugs) > 1:

            comment += "s"

        comment += " " + ", ".join('#' + b for b in self.bugs) + ")"

        if self.block_by_regression:

            comment += ', blocks {} by autopkgtest regression'.format(', '.join(sorted(self.block_by_regression)))

        if self.block_by_uninstallability:

            comment += ', blocks {} by uninstallability'.format(', '.join(sorted(self.block_by_uninstallability)))

        return comment

    @property

    def rdeps_text_short(self):

        return "\n".join(self.reverse_depends()[:10])

    @property

    def rdeps_text_more(self):

        return "\n".join(self.reverse_depends()[10:])

    @property

    def removal_commands(self):

        suites = self.suites

        if not suites:

            suites = [series]

        cmds = []

        for suite in suites:

            cmd = " ".join(["remove-package", "-s", suite, "-y", "-m", shlex.quote(self.comment), self.source_package_name])

            if self.reverse_depends():

                cmd = '#' + cmd

            cmds.append(cmd)

        return cmds

    @property

    def css_class(self):

        if self.reverse_depends():

            return 'removal-not-ok'

        else:

            return 'removal-ok'

if 'SERIES' in os.environ:

    series = os.environ['SERIES']

else:

    series = run_output('distro-info', '-d')

def main():

    print("loading data")

    if args.ubuntu_excuses.endswith('.xz'):

        import lzma

        excuses_opener = lzma.open

    else:

        excuses_opener = open

    with excuses_opener(args.ubuntu_excuses) as fp:

        ubuntu_excuses = yaml.load(fp, Loader=yaml.CSafeLoader)

    with open(args.ubuntu_update_output) as fp:

        bin_pkg_arch_to_blocked_src_pkgs = extract_bin_pkg_arch_to_blocked_src_pkgs(fp)

    with open(args.debian_excuses) as fp:

        debian_excuses = yaml.load(fp, Loader=yaml.CSafeLoader)

    print("finding rcgone packages")

    rc_gones = {}

    for source in debian_excuses['sources']:

        if source['old-version'] == '-':

            info = source['policy_info']

            rc_bugs = info.get('rc-bugs', {})

            if rc_bugs.get('verdict') == "REJECTED_PERMANENTLY":

                bugs = []

                for k in 'shared-bugs', 'unique-source-bugs', 'unique-target-bugs':

                    bugs.extend(rc_bugs[k])

                rc_gones[source['item-name']] = RCGone(

                    source_package_name=source['item-name'],

                    bugs=bugs)

    in_proposed_by_autopkgtest_or_missing_binaries = set()

    print("checking autopkgtests")

    for source in ubuntu_excuses['sources']:

        item = source['item-name']

        if 'autopkgtest' in source['reason']:

            in_proposed_by_autopkgtest_or_missing_binaries.add(item)

            for package, results in sorted(source['policy_info']['autopkgtest'].items()):

                if '/' not in package:

                    # This only happens when all tests are still running

                    continue

                package, version = package.split('/')

                if package not in rc_gones or '-0ubuntu' in version:

                    continue

                for arch, result in sorted(results.items()):

                    outcome, log, history, wtf1, wtf2 = result

                    if outcome == "REGRESSION" and package != item:

                        rc_gones[package].block_by_regression.add(item)

                        break

        if 'missing-builds' in source and '-0ubuntu' not in source['new-version']:

            in_proposed_by_autopkgtest_or_missing_binaries.add(item)

        if item in rc_gones:

            if source['new-version'] != '-':

                rc_gones[item].suites.add(series+"-proposed")

            if source['old-version'] != '-':

                rc_gones[item].suites.add(series)

    print("checking uninstallability")

    for rc_gone in rc_gones.values():

        if rc_gone.source_package_name not in in_proposed_by_autopkgtest_or_missing_binaries:

            continue

        for bin_pkg, arch in set(rc_gone.binary_pkgs()):

            rc_gone.block_by_uninstallability.update(bin_pkg_arch_to_blocked_src_pkgs.get((bin_pkg, arch), set()))

    print("finding reverse-deps")

    packages = []

    for _, rc_gone in sorted(rc_gones.items()):

        if not rc_gone.block_by_regression and not rc_gone.block_by_uninstallability:

            continue

        rc_gone.reverse_depends()

        packages.append(rc_gone)

    print("rendering")

    t = env.get_template('rcbuggy-problem-packages.html')

    with open(args.output, 'w', encoding='utf-8') as fp:

        fp.write(t.render(

            packages=packages,

            now=ubuntu_excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S")))

if __name__ == '__main__':

    main()