← Back to branch summary

~ubuntu-archive/ubuntu-archive-scripts/trunk

« back to all changes in this revision

Viewing changes to generate-team-p-m

  • Committer: Adam Conrad
  • Date: 2019-10-18 09:21:50 UTC
  • Revision ID: adconrad@0c3.net-20191018092150-hczsq3uomg7hqroo
auto-accept: Commit production changes.

Show diffs side-by-side

added added

removed removed

Lines of Context:
generate-team-p-m
18
18
import argparse
19
19
from collections import defaultdict, OrderedDict
20
20
import datetime
21
 
import time
22
21
import json
23
22
import os
24
23
import threading
25
24
from urllib.request import urlopen
26
 
import urllib.error
27
25
 
28
26
import attr
29
27
from jinja2 import Environment, FileSystemLoader
30
28
import yaml
31
 
import lzma
32
29
 
33
30
env = Environment(
34
31
    loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),
125
122
 
126
123
def get_subscribers_json(packages, subscribers_json):
127
124
    if subscribers_json is None:
128
 
        j = urlopen("https://ubuntu-archive-team.ubuntu.com/package-team-mapping.json")
 
125
        j = urlopen("http://people.canonical.com/~ubuntu-archive/package-team-mapping.json")
129
126
    else:
130
127
        j = open(subscribers_json, 'rb')
131
128
    with j:
191
188
    waiting = attr.ib(default=None) # [(source_package_name, arches)]
192
189
    data = attr.ib(default=None) # data for package_in_proposed
193
190
    unsatdepends = attr.ib(default=None) # [string]
194
 
    unsatbuilddep = attr.ib(default=None) # [string]
195
 
    brokenbin = attr.ib(default=None) # [string]
196
 
    componentmismatch = attr.ib(default=None) # [string]
197
191
 
198
192
    _age = attr.ib(default=None)
199
193
 
211
205
        if self._age is not None:
212
206
            return self._age
213
207
        else:
214
 
            try:
215
 
                return self.data["policy_info"]["age"]["current-age"]
216
 
            except KeyError:
217
 
                return -1
 
208
            return self.data["policy_info"]["age"]["current-age"]
218
209
 
219
210
    @age.setter
220
211
    def age(self, val):
243
234
 
244
235
    print("fetching yaml")
245
236
    if args.excuses_yaml:
246
 
        if args.excuses_yaml.endswith('.xz'):
247
 
            yaml_text = lzma.open(args.excuses_yaml)
248
 
        else:
249
 
            yaml_text = open(args.excuses_yaml)
 
237
        yaml_text = open(args.excuses_yaml).read()
250
238
    else:
251
 
        try:
252
 
            yaml_text = lzma.open(urlopen("https://ubuntu-archive-team.ubuntu.com/proposed-migration/update_excuses.yaml.xz"))
253
 
        except urllib.error.HTTPError as e:
254
 
            print("Reading fallback yaml (%s)" % e)
255
 
            yaml_text = urlopen("https://ubuntu-archive-team.ubuntu.com/proposed-migration/update_excuses.yaml")
 
239
        yaml_text = urlopen("https://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.yaml").read()
256
240
    print("parsing yaml")
257
241
    # The CSafeLoader is ten times faster than the regular one
258
242
    excuses = yaml.load(yaml_text, Loader=yaml.CSafeLoader)
264
248
        # Missing component means main
265
249
        if item.get('component', 'main') not in components:
266
250
            continue
267
 
        prob = Problem(kind='package-in-proposed', data=defaultdict(dict, item), package_in_proposed=source_package_name)
 
251
        prob = Problem(kind='package-in-proposed', data=item, package_in_proposed=source_package_name)
268
252
        in_proposed_packages[source_package_name] = prob
269
253
        prob.regressions = []
270
254
        prob.waiting = []
271
 
        prob.componentmismatch = []
272
 
        # The verdict entries are not items to list on the report
273
 
        for policy in ['autopkgtest', 'update-excuse', 'block-bugs']:
274
 
            try:
275
 
                del item['policy_info'][policy]['verdict']
276
 
            except KeyError:
277
 
                pass
278
255
        if 'autopkgtest' in item['reason']:
279
256
            for package, results in sorted(item['policy_info']['autopkgtest'].items()):
280
257
                regr_arches = []
292
269
                    prob.regressions.append(regr)
293
270
                if wait_arches:
294
271
                    prob.waiting.append((package + ": " + ", ".join(wait_arches)))
295
 
        if 'depends' in item['reason']:
296
 
            for l in item['excuses']:
297
 
                if 'cannot depend on' in l:
298
 
                    prob.componentmismatch.append(l)
299
272
        if 'dependencies' in item and 'unsatisfiable-dependencies' in item['dependencies']:
300
273
                unsatd = defaultdict(list)
301
274
                for arch, packages in item['dependencies']['unsatisfiable-dependencies'].items():
302
275
                    for p in packages:
303
276
                        unsatd[p].append(arch)
304
277
                prob.unsatdepends = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatd.items())]
305
 
        if 'policy_info' in item:
306
 
            if 'build-depends' in item['policy_info'] and 'unsatisfiable-arch-build-depends' in item['policy_info']['build-depends']:
307
 
                    unsatdbd = defaultdict(list)
308
 
                    for arch, packages in item['policy_info']['build-depends']['unsatisfiable-arch-build-depends'].items():
309
 
                        for p in packages:
310
 
                            unsatdbd[p].append(arch)
311
 
                    prob.unsatbuilddep = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatdbd.items())]
312
 
            if 'implicit-deps' in item['policy_info']['implicit-deps']:
313
 
                    prob.brokenbin = item['policy_info']['implicit-deps']['implicit-deps']['broken-binaries']
314
278
 
315
279
    package_to_problems = defaultdict(list)
316
280
 
317
281
    for problem in in_proposed_packages.values():
318
 
        # nautilus/riscv64 -> nautilus
319
 
        pkg = problem.package_in_proposed.split('/')[0]
320
 
        package_to_problems[pkg].append(problem)
 
282
        package_to_problems[problem.package_in_proposed].append(problem)
321
283
        for regression in problem.regressions:
322
284
            if regression.blocking not in in_proposed_packages:
323
285
                continue
344
306
    else:
345
307
        subscribers = get_subscribers_json(set(package_to_problems), args.subscribers_json)
346
308
        for p in set(package_to_problems):
347
 
            pkg = p.split('/')[0]
348
 
            if pkg not in subscribers:
 
309
            if p not in subscribers:
349
310
                subscribers[p] = ['unknown']
350
311
 
351
312
    all_teams = set()
369
330
            all_teams=all_teams,
370
331
            team_to_problems=team_to_problems,
371
332
            team_to_attn_count=team_to_attn_count,
372
 
            now=excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S") + ' ' + time.localtime().tm_zone))
 
333
            now=excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S")))
373
334
    if args.yaml_output:
374
335
        team_to_problem_data = {}
375
336
        for t, ps in team_to_problems.items():