~ubuntu-archive/ubuntu-archive-scripts/trunk : revision 324

1

#!/usr/bin/env python3

2

# -*- coding: utf-8 -*-

3

4

5

6

# This program is free software; you can redistribute it and/or

7

# modify it under the terms of the GNU General Public License

8

# as published by the Free Software Foundation; either version 2

9

# of the License, or (at your option) any later version.

10

#

11

# This program is distributed in the hope that it will be useful,

12

# but WITHOUT ANY WARRANTY; without even the implied warranty of

13

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

14

# GNU General Public License for more details.

15

#

16

# A copy of the GNU General Public License version 2 is in LICENSE.

17

18

import argparse

19

from collections import defaultdict, OrderedDict

20

import datetime

21

import time

22

import json

23

import os

24

import threading

25

from urllib.request import urlopen

26

import urllib.error

27

28

import attr

29

from jinja2 import Environment, FileSystemLoader

30

import yaml

31

import lzma

32

33

env = Environment(

34

loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),

35

autoescape=True,

36

extensions=['jinja2.ext.i18n'],

37

)

38

env.install_null_translations(True)

39

40

_lps = {}

41

42

def get_lp(i, anon=True):

43

from launchpadlib.launchpad import Launchpad

44

k = (i, anon)

45

if k not in _lps:

46

print(i, "connecting...")

47

if anon:

48

_lps[k] = Launchpad.login_anonymously('sru-team-report', 'production', version='devel')

49

else:

50

_lps[k] = Launchpad.login_with('sru-team-report', 'production', version='devel')

51

return _lps[k]

52

53

54

def get_true_ages_in_proposed(package_names, thread_count):

55

package_names = set(package_names)

56

r = {}

57

def run(i):

58

lp = get_lp(i, True)

59

ubuntu = lp.distributions['ubuntu']

60

primary_archive = ubuntu.archives[0]

61

devel_series = ubuntu.getSeries(name_or_version=ubuntu.development_series_alias)

62

while True:

63

try:

64

spn = package_names.pop()

65

except KeyError:

66

return

67

print(i, "getting true age in proposed for", spn)

68

history = primary_archive.getPublishedSources(

69

source_name=spn, distro_series=devel_series, exact_match=True)

70

last_proposed_spph = None

71

for spph in history:

72

if spph.pocket != "Proposed":

73

break

74

last_proposed_spph = spph

75

if last_proposed_spph is None:

76

continue

77

age = datetime.datetime.now(tz=last_proposed_spph.date_created.tzinfo) - last_proposed_spph.date_created

78

r[spn] = age.total_seconds() / 3600 / 24

79

threads = []

80

for i in range(thread_count):

81

t = threading.Thread(target=run, args=(i,))

82

threads.append(t)

83

t.start()

84

for t in threads:

85

t.join()

86

return r

87

88

89

def get_subscribers_lp(packages, thread_count):

90

from lazr.restfulclient.errors import ClientError

91

packages = set(packages)

92

def run(i, subscribers):

93

lp = get_lp(i, False)

94

ubuntu = lp.distributions['ubuntu']

95

while True:

96

try:

97

spn = packages.pop()

98

except KeyError:

99

return

100

print(i, spn)

101

distribution_source_package = ubuntu.getSourcePackage(name=spn)

102

for subscription in distribution_source_package.getSubscriptions():

103

subscriber = subscription.subscriber

104

try:

105

if subscriber.is_team:

106

subscribers[spn].append(subscriber.name)

107

except ClientError:

108

# This happens for suspended users

109

pass

110

results = []

111

threads = []

112

for i in range(thread_count):

113

d = defaultdict(list)

114

t = threading.Thread(target=run, args=(i, d))

115

results.append(d)

116

threads.append(t)

117

t.start()

118

for t in threads:

119

t.join()

120

result = defaultdict(list)

121

for d in results:

122

for k, v in d.items():

123

result[k].extend(v)

124

return result

125

126

def get_subscribers_json(packages, subscribers_json):

127

if subscribers_json is None:

128

j = urlopen("http://people.canonical.com/~ubuntu-archive/package-team-mapping.json")

129

else:

130

j = open(subscribers_json, 'rb')

131

with j:

132

team_to_packages = json.loads(j.read().decode('utf-8'))

133

package_to_teams = {}

134

for team, packages in team_to_packages.items():

135

for package in packages:

136

package_to_teams.setdefault(package, []).append(team)

137

return package_to_teams

138

139

def setup_yaml():

140

""" http://stackoverflow.com/a/8661021 """

141

represent_dict_order = (

142

lambda self, data: self.represent_mapping('tag:yaml.org,2002:map',

143

data.items()))

144

yaml.add_representer(OrderedDict, represent_dict_order)

145

146

147

setup_yaml()

148

149

def as_data(inst):

150

r = OrderedDict()

151

fields = [field.name for field in attr.fields(type(inst))]

152

fields.extend(getattr(inst, "extra_fields", []))

153

for field in fields:

154

if field.startswith('_'):

155

continue

156

v = getattr(

157

inst,

158

'serialize_' + field,

159

lambda: getattr(inst, field))()

160

if v is not None:

161

r[field] = v

162

return r

163

164

@attr.s

165

class ArchRegression:

166

arch = attr.ib(default=None)

167

log_link = attr.ib(default=None)

168

hist_link = attr.ib(default=None)

169

170

171

@attr.s

172

class Regression:

173

blocking = attr.ib(default=None) # source package name blocked

174

package = attr.ib(default=None) # source package name doing the blocking

175

version = attr.ib(default=None) # version that regressed

176

arches = attr.ib(default=None) # [ArchRegression]

177

178

def serialize_arches(self):

179

return [as_data(a) for a in self.arches]

180

181

@property

182

def package_version(self):

183

return self.package + '/' + self.version

184

185

@attr.s

186

class Problem:

187

kind = attr.ib(default=None) # 'blocked-in-proposed', 'regressing-other'

188

package_in_proposed = attr.ib(default=None) # name of package that's in proposed

189

regressing_package = attr.ib(default=None) # name of package regressing package_in_proposed, None if blocked-in-proposed

190

regressions = attr.ib(default=None) # [Regression]

191

waiting = attr.ib(default=None) # [(source_package_name, arches)]

192

data = attr.ib(default=None) # data for package_in_proposed

193

unsatdepends = attr.ib(default=None) # [string]

194

unsatbuilddep = attr.ib(default=None) # [string]

195

brokenbin = attr.ib(default=None) # [string]

196

componentmismatch = attr.ib(default=None) # [string]

197

198

_age = attr.ib(default=None)

199

200

extra_fields = ['age']

201

202

def serialize_regressions(self):

203

return [as_data(r) for r in self.regressions]

204

205

@property

206

def late(self):

207

return self.age > 3

208

209

@property

210

def age(self):

211

if self._age is not None:

212

return self._age

213

else:

214

try:

215

return self.data["policy_info"]["age"]["current-age"]

216

except KeyError:

217

return -1

218

219

@age.setter

220

def age(self, val):

221

self._age = val

222

223

@property

224

def key_package(self):

225

if self.regressing_package:

226

return self.regressing_package

227

return self.package_in_proposed

228

229

230

def main():

231

parser = argparse.ArgumentParser()

232

parser.add_argument('--ppa', action='store')

233

parser.add_argument('--components', action='store', default="main,restricted")

234

parser.add_argument('--subscribers-from-lp', action='store_true')

235

parser.add_argument('--subscribers-json', action='store')

236

parser.add_argument('--true-ages', action='store_true')

237

parser.add_argument('--excuses-yaml', action='store')

238

parser.add_argument('output')

239

parser.add_argument('yaml_output', default=None)

240

args = parser.parse_args()

241

242

components = args.components.split(',')

243

244

print("fetching yaml")

245

if args.excuses_yaml:

246

if args.excuses_yaml.endswith('.xz'):

247

yaml_text = lzma.open(args.excuses_yaml)

248

else:

249

yaml_text = open(args.excuses_yaml)

250

else:

251

try:

252

yaml_text = lzma.open(urlopen("https://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.yaml.xz"))

253

except urllib.error.HTTPError as e:

254

print("Reading fallback yaml (%s)" % e)

255

yaml_text = urlopen("https://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.yaml")

256

print("parsing yaml")

257

# The CSafeLoader is ten times faster than the regular one

258

excuses = yaml.load(yaml_text, Loader=yaml.CSafeLoader)

259

260

print("pre-processing packages")

261

in_proposed_packages = {}

262

for item in excuses["sources"]:

263

source_package_name = item['item-name']

264

# Missing component means main

265

if item.get('component', 'main') not in components:

266

continue

267

prob = Problem(kind='package-in-proposed', data=defaultdict(dict, item), package_in_proposed=source_package_name)

268

in_proposed_packages[source_package_name] = prob

269

prob.regressions = []

270

prob.waiting = []

271

prob.componentmismatch = []

272

# The verdict entries are not items to list on the report

273

for policy in ['autopkgtest', 'update-excuse', 'block-bugs']:

274

try:

275

del item['policy_info'][policy]['verdict']

276

except KeyError:

277

pass

278

if 'autopkgtest' in item['reason']:

279

for package, results in sorted(item['policy_info']['autopkgtest'].items()):

280

regr_arches = []

281

wait_arches = []

282

for arch, result in sorted(results.items()):

283

outcome, log, history, wtf1, wtf2 = result

284

if outcome == "REGRESSION":

285

regr_arches.append(ArchRegression(arch=arch, log_link=log, hist_link=history))

286

if outcome == "RUNNING":

287

wait_arches.append(arch)

288

if regr_arches:

289

p, v = package.split('/')

290

regr = Regression(package=p, version=v, blocking=source_package_name)

291

regr.arches = regr_arches

292

prob.regressions.append(regr)

293

if wait_arches:

294

prob.waiting.append((package + ": " + ", ".join(wait_arches)))

295

if 'depends' in item['reason']:

296

for l in item['excuses']:

297

if 'cannot depend on' in l:

298

prob.componentmismatch.append(l)

299

if 'dependencies' in item and 'unsatisfiable-dependencies' in item['dependencies']:

300

unsatd = defaultdict(list)

301

for arch, packages in item['dependencies']['unsatisfiable-dependencies'].items():

302

for p in packages:

303

unsatd[p].append(arch)

304

prob.unsatdepends = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatd.items())]

305

if 'policy_info' in item:

306

if 'build-depends' in item['policy_info'] and 'unsatisfiable-arch-build-depends' in item['policy_info']['build-depends']:

307

unsatdbd = defaultdict(list)

308

for arch, packages in item['policy_info']['build-depends']['unsatisfiable-arch-build-depends'].items():

309

for p in packages:

310

unsatdbd[p].append(arch)

311

prob.unsatbuilddep = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatdbd.items())]

312

if 'implicit-deps' in item['policy_info']['implicit-deps']:

313

prob.brokenbin = item['policy_info']['implicit-deps']['implicit-deps']['broken-binaries']

314

315

package_to_problems = defaultdict(list)

316

317

for problem in in_proposed_packages.values():

318

# nautilus/riscv64 -> nautilus

319

pkg = problem.package_in_proposed.split('/')[0]

320

package_to_problems[pkg].append(problem)

321

for regression in problem.regressions:

322

if regression.blocking not in in_proposed_packages:

323

continue

324

if regression.blocking == regression.package:

325

continue

326

package_to_problems[regression.package].append(Problem(

327

kind='regressing-other', package_in_proposed=regression.blocking,

328

regressing_package=regression.package,

329

regressions=[regression],

330

data=in_proposed_packages[regression.blocking].data))

331

332

if args.true_ages:

333

true_ages = get_true_ages_in_proposed(set(package_to_problems), 10)

334

for package, true_age in true_ages.items():

335

for problem in package_to_problems[package]:

336

problem.age = true_age

337

338

print("getting subscribers")

339

if args.subscribers_from_lp:

340

subscribers = get_subscribers_lp(set(package_to_problems), 10)

341

for p in set(package_to_problems):

342

if p not in subscribers:

343

subscribers[p] = ['unsubscribed']

344

else:

345

subscribers = get_subscribers_json(set(package_to_problems), args.subscribers_json)

346

for p in set(package_to_problems):

347

pkg = p.split('/')[0]

348

if pkg not in subscribers:

349

subscribers[p] = ['unknown']

350

351

all_teams = set()

352

team_to_problems = defaultdict(list)

353

for package, teams in subscribers.items():

354

all_teams |= set(teams)

355

for team in teams:

356

team_to_problems[team].extend(package_to_problems[package])

357

358

for packages in team_to_problems.values():

359

packages.sort(key=lambda prob: (-prob.age, prob.key_package))

360

361

team_to_attn_count = {}

362

for team, problems in team_to_problems.items():

363

team_to_attn_count[team] = len([problem for problem in problems if problem.late])

364

365

print("rendering")

366

t = env.get_template('team-report.html')

367

with open(args.output, 'w', encoding='utf-8') as fp:

368

fp.write(t.render(

369

all_teams=all_teams,

370

team_to_problems=team_to_problems,

371

team_to_attn_count=team_to_attn_count,

372

now=excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S") + ' ' + time.localtime().tm_zone))

373

if args.yaml_output:

374

team_to_problem_data = {}

375

for t, ps in team_to_problems.items():

376

team_to_problem_data[t] = [as_data(p) for p in ps]

377

with open(args.yaml_output, 'w', encoding='utf-8') as fp:

378

yaml.dump(team_to_problem_data, fp)

379

380

381

if __name__ == '__main__':

382

main()