~ubuntu-archive/ubuntu-archive-scripts/trunk

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright (C) 2018 Canonical Ltd

# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# A copy of the GNU General Public License version 2 is in LICENSE.

import argparse
from collections import defaultdict, OrderedDict
import datetime
import time
import json
import os
import threading
from urllib.request import urlopen
import urllib.error

import attr
from jinja2 import Environment, FileSystemLoader
import yaml
import lzma

env = Environment(
    loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),
    autoescape=True,
    extensions=['jinja2.ext.i18n'],
)
env.install_null_translations(True)

_lps = {}

def get_lp(i, anon=True):
    from launchpadlib.launchpad import Launchpad
    k = (i, anon)
    if k not in _lps:
        print(i, "connecting...")
        if anon:
            _lps[k] = Launchpad.login_anonymously('sru-team-report', 'production', version='devel')
        else:
            _lps[k] = Launchpad.login_with('sru-team-report', 'production', version='devel')
    return _lps[k]


def get_true_ages_in_proposed(package_names, thread_count):
    package_names = set(package_names)
    r = {}
    def run(i):
        lp = get_lp(i, True)
        ubuntu = lp.distributions['ubuntu']
        primary_archive = ubuntu.archives[0]
        devel_series = ubuntu.getSeries(name_or_version=ubuntu.development_series_alias)
        while True:
            try:
                spn = package_names.pop()
            except KeyError:
                return
            print(i, "getting true age in proposed for", spn)
            history = primary_archive.getPublishedSources(
                source_name=spn, distro_series=devel_series, exact_match=True)
            last_proposed_spph = None
            for spph in history:
                if spph.pocket != "Proposed":
                    break
                last_proposed_spph = spph
            if last_proposed_spph is None:
                continue
            age = datetime.datetime.now(tz=last_proposed_spph.date_created.tzinfo) - last_proposed_spph.date_created
            r[spn] = age.total_seconds() / 3600 / 24
    threads = []
    for i in range(thread_count):
        t = threading.Thread(target=run, args=(i,))
        threads.append(t)
        t.start()
    for t in threads:
        t.join()
    return r


def get_subscribers_lp(packages, thread_count):
    from lazr.restfulclient.errors import ClientError
    packages = set(packages)
    def run(i, subscribers):
        lp = get_lp(i, False)
        ubuntu = lp.distributions['ubuntu']
        while True:
            try:
                spn = packages.pop()
            except KeyError:
                return
            print(i, spn)
            distribution_source_package = ubuntu.getSourcePackage(name=spn)
            for subscription in distribution_source_package.getSubscriptions():
                subscriber = subscription.subscriber
                try:
                    if subscriber.is_team:
                        subscribers[spn].append(subscriber.name)
                except ClientError:
                    # This happens for suspended users
                    pass
    results = []
    threads = []
    for i in range(thread_count):
        d = defaultdict(list)
        t = threading.Thread(target=run, args=(i, d))
        results.append(d)
        threads.append(t)
        t.start()
    for t in threads:
        t.join()
    result = defaultdict(list)
    for d in results:
        for k, v in d.items():
            result[k].extend(v)
    return result

def get_subscribers_json(packages, subscribers_json):
    if subscribers_json is None:
        j = urlopen("https://ubuntu-archive-team.ubuntu.com/package-team-mapping.json")
    else:
        j = open(subscribers_json, 'rb')
    with j:
        team_to_packages = json.loads(j.read().decode('utf-8'))
    package_to_teams = {}
    for team, packages in team_to_packages.items():
        for package in packages:
            package_to_teams.setdefault(package, []).append(team)
    return package_to_teams

def setup_yaml():
    """ http://stackoverflow.com/a/8661021 """
    represent_dict_order = (
        lambda self, data: self.represent_mapping('tag:yaml.org,2002:map',
                                                  data.items()))
    yaml.add_representer(OrderedDict, represent_dict_order)


setup_yaml()

def as_data(inst):
    r = OrderedDict()
    fields = [field.name for field in attr.fields(type(inst))]
    fields.extend(getattr(inst, "extra_fields", []))
    for field in fields:
        if field.startswith('_'):
            continue
        v = getattr(
            inst,
            'serialize_' + field,
            lambda: getattr(inst, field))()
        if v is not None:
            r[field] = v
    return r

@attr.s
class ArchRegression:
    arch = attr.ib(default=None)
    log_link = attr.ib(default=None)
    hist_link = attr.ib(default=None)


@attr.s
class Regression:
    blocking = attr.ib(default=None) # source package name blocked
    package = attr.ib(default=None) # source package name doing the blocking
    version = attr.ib(default=None) # version that regressed
    arches = attr.ib(default=None) # [ArchRegression]

    def serialize_arches(self):
        return [as_data(a) for a in self.arches]

    @property
    def package_version(self):
        return self.package + '/' + self.version

@attr.s
class Problem:
    kind = attr.ib(default=None) # 'blocked-in-proposed', 'regressing-other'
    package_in_proposed = attr.ib(default=None) # name of package that's in proposed
    regressing_package = attr.ib(default=None) # name of package regressing package_in_proposed, None if blocked-in-proposed
    regressions = attr.ib(default=None) # [Regression]
    waiting = attr.ib(default=None) # [(source_package_name, arches)]
    data = attr.ib(default=None) # data for package_in_proposed
    unsatdepends = attr.ib(default=None) # [string]
    unsatbuilddep = attr.ib(default=None) # [string]
    brokenbin = attr.ib(default=None) # [string]
    componentmismatch = attr.ib(default=None) # [string]

    _age = attr.ib(default=None)

    extra_fields = ['age']

    def serialize_regressions(self):
        return [as_data(r) for r in self.regressions]

    @property
    def late(self):
        return self.age > 3

    @property
    def age(self):
        if self._age is not None:
            return self._age
        else:
            try:
                return self.data["policy_info"]["age"]["current-age"]
            except KeyError:
                return -1

    @age.setter
    def age(self, val):
        self._age = val

    @property
    def key_package(self):
        if self.regressing_package:
            return self.regressing_package
        return self.package_in_proposed


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--ppa', action='store')
    parser.add_argument('--components', action='store', default="main,restricted")
    parser.add_argument('--subscribers-from-lp', action='store_true')
    parser.add_argument('--subscribers-json', action='store')
    parser.add_argument('--true-ages', action='store_true')
    parser.add_argument('--excuses-yaml', action='store')
    parser.add_argument('output')
    parser.add_argument('yaml_output', default=None)
    args = parser.parse_args()

    components = args.components.split(',')

    print("fetching yaml")
    if args.excuses_yaml:
        if args.excuses_yaml.endswith('.xz'):
            yaml_text = lzma.open(args.excuses_yaml)
        else:
            yaml_text = open(args.excuses_yaml)
    else:
        try:
            yaml_text = lzma.open(urlopen("https://ubuntu-archive-team.ubuntu.com/proposed-migration/update_excuses.yaml.xz"))
        except urllib.error.HTTPError as e:
            print("Reading fallback yaml (%s)" % e)
            yaml_text = urlopen("https://ubuntu-archive-team.ubuntu.com/proposed-migration/update_excuses.yaml")
    print("parsing yaml")
    # The CSafeLoader is ten times faster than the regular one
    excuses = yaml.load(yaml_text, Loader=yaml.CSafeLoader)

    print("pre-processing packages")
    in_proposed_packages = {}
    for item in excuses["sources"]:
        source_package_name = item['item-name']
        # Missing component means main
        if item.get('component', 'main') not in components:
            continue
        prob = Problem(kind='package-in-proposed', data=defaultdict(dict, item), package_in_proposed=source_package_name)
        in_proposed_packages[source_package_name] = prob
        prob.regressions = []
        prob.waiting = []
        prob.componentmismatch = []
        # The verdict entries are not items to list on the report
        for policy in ['autopkgtest', 'update-excuse', 'block-bugs']:
            try:
                del item['policy_info'][policy]['verdict']
            except KeyError:
                pass
        if 'autopkgtest' in item['reason']:
            for package, results in sorted(item['policy_info']['autopkgtest'].items()):
                regr_arches = []
                wait_arches = []
                for arch, result in sorted(results.items()):
                    outcome, log, history, wtf1, wtf2 = result
                    if outcome == "REGRESSION":
                        regr_arches.append(ArchRegression(arch=arch, log_link=log, hist_link=history))
                    if outcome == "RUNNING":
                        wait_arches.append(arch)
                if regr_arches:
                    p, v = package.split('/')
                    regr = Regression(package=p, version=v, blocking=source_package_name)
                    regr.arches = regr_arches
                    prob.regressions.append(regr)
                if wait_arches:
                    prob.waiting.append((package + ": " + ", ".join(wait_arches)))
        if 'depends' in item['reason']:
            for l in item['excuses']:
                if 'cannot depend on' in l:
                    prob.componentmismatch.append(l)
        if 'dependencies' in item and 'unsatisfiable-dependencies' in item['dependencies']:
                unsatd = defaultdict(list)
                for arch, packages in item['dependencies']['unsatisfiable-dependencies'].items():
                    for p in packages:
                        unsatd[p].append(arch)
                prob.unsatdepends = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatd.items())]
        if 'policy_info' in item:
            if 'build-depends' in item['policy_info'] and 'unsatisfiable-arch-build-depends' in item['policy_info']['build-depends']:
                    unsatdbd = defaultdict(list)
                    for arch, packages in item['policy_info']['build-depends']['unsatisfiable-arch-build-depends'].items():
                        for p in packages:
                            unsatdbd[p].append(arch)
                    prob.unsatbuilddep = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatdbd.items())]

    package_to_problems = defaultdict(list)

    for problem in in_proposed_packages.values():
        # nautilus/riscv64 -> nautilus
        pkg = problem.package_in_proposed.split('/')[0]
        package_to_problems[pkg].append(problem)
        for regression in problem.regressions:
            if regression.blocking not in in_proposed_packages:
                continue
            if regression.blocking == regression.package:
                continue
            package_to_problems[regression.package].append(Problem(
                kind='regressing-other', package_in_proposed=regression.blocking,
                regressing_package=regression.package,
                regressions=[regression],
                data=in_proposed_packages[regression.blocking].data))

    if args.true_ages:
        true_ages = get_true_ages_in_proposed(set(package_to_problems), 10)
        for package, true_age in true_ages.items():
            for problem in package_to_problems[package]:
                problem.age = true_age

    print("getting subscribers")
    if args.subscribers_from_lp:
        subscribers = get_subscribers_lp(set(package_to_problems), 10)
        for p in set(package_to_problems):
            if p not in subscribers:
                subscribers[p] = ['unsubscribed']
    else:
        subscribers = get_subscribers_json(set(package_to_problems), args.subscribers_json)
        for p in set(package_to_problems):
            pkg = p.split('/')[0]
            if pkg not in subscribers:
                subscribers[p] = ['unknown']

    all_teams = set()
    team_to_problems = defaultdict(list)
    for package, teams in subscribers.items():
        all_teams |= set(teams)
        for team in teams:
            team_to_problems[team].extend(package_to_problems[package])

    for packages in team_to_problems.values():
        packages.sort(key=lambda prob: (-prob.age, prob.key_package))

    team_to_attn_count = {}
    for team, problems in team_to_problems.items():
        team_to_attn_count[team] = len([problem for problem in problems if problem.late])

    print("rendering")
    t = env.get_template('team-report.html')
    with open(args.output, 'w', encoding='utf-8') as fp:
        fp.write(t.render(
            all_teams=all_teams,
            team_to_problems=team_to_problems,
            team_to_attn_count=team_to_attn_count,
            now=excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S") + ' ' + time.localtime().tm_zone))
    if args.yaml_output:
        team_to_problem_data = {}
        for t, ps in team_to_problems.items():
            team_to_problem_data[t] = [as_data(p) for p in ps]
        with open(args.yaml_output, 'w', encoding='utf-8') as fp:
            yaml.dump(team_to_problem_data, fp)


if __name__ == '__main__':
    main()