~brian-murray/ubuntu-archive-scripts/remove-vintage-packages : revision 225.1.1

1

#!/usr/bin/env python3

2

# -*- coding: utf-8 -*-

3

4

5

6

# This program is free software; you can redistribute it and/or

7

# modify it under the terms of the GNU General Public License

8

# as published by the Free Software Foundation; either version 2

9

# of the License, or (at your option) any later version.

10

#

11

# This program is distributed in the hope that it will be useful,

12

# but WITHOUT ANY WARRANTY; without even the implied warranty of

13

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

14

# GNU General Public License for more details.

15

#

16

# A copy of the GNU General Public License version 2 is in LICENSE.

17

18

import argparse

19

from collections import defaultdict

20

import datetime

21

import json

22

import os

23

import threading

24

from urllib.request import urlopen

25

26

import attr

27

from jinja2 import Environment, FileSystemLoader

28

import yaml

29

30

env = Environment(

31

loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),

32

autoescape=True,

33

extensions=['jinja2.ext.i18n'],

34

)

35

env.install_null_translations(True)

36

37

_lps = {}

38

39

def get_lp(i, anon=True):

40

from launchpadlib.launchpad import Launchpad

41

k = (i, anon)

42

if k not in _lps:

43

print(i, "connecting...")

44

if anon:

45

_lps[k] = Launchpad.login_anonymously('sru-team-report', 'production', version='devel')

46

else:

47

_lps[k] = Launchpad.login_with('sru-team-report', 'production', version='devel')

48

return _lps[k]

49

50

51

def get_true_ages_in_proposed(package_names, thread_count):

52

package_names = set(package_names)

53

r = {}

54

def run(i):

55

lp = get_lp(i, True)

56

ubuntu = lp.distributions['ubuntu']

57

primary_archive = ubuntu.archives[0]

58

devel_series = ubuntu.getSeries(name_or_version=ubuntu.development_series_alias)

59

while True:

60

try:

61

spn = package_names.pop()

62

except KeyError:

63

return

64

print(i, "getting true age in proposed for", spn)

65

history = primary_archive.getPublishedSources(

66

source_name=spn, distro_series=devel_series, exact_match=True)

67

last_proposed_spph = None

68

for spph in history:

69

if spph.pocket != "Proposed":

70

break

71

last_proposed_spph = spph

72

if last_proposed_spph is None:

73

continue

74

age = datetime.datetime.now(tz=last_proposed_spph.date_created.tzinfo) - last_proposed_spph.date_created

75

r[spn] = age.total_seconds() / 3600 / 24

76

threads = []

77

for i in range(thread_count):

78

t = threading.Thread(target=run, args=(i,))

79

threads.append(t)

80

t.start()

81

for t in threads:

82

t.join()

83

return r

84

85

86

def get_subscribers_lp(packages, thread_count):

87

from lazr.restfulclient.errors import ClientError

88

packages = set(packages)

89

def run(i, subscribers):

90

lp = get_lp(i, False)

91

ubuntu = lp.distributions['ubuntu']

92

while True:

93

try:

94

spn = packages.pop()

95

except KeyError:

96

return

97

print(i, spn)

98

distribution_source_package = ubuntu.getSourcePackage(name=spn)

99

for subscription in distribution_source_package.getSubscriptions():

100

subscriber = subscription.subscriber

101

try:

102

if subscriber.is_team:

103

subscribers[spn].append(subscriber.name)

104

except ClientError:

105

# This happens for suspended users

106

pass

107

results = []

108

threads = []

109

for i in range(thread_count):

110

d = defaultdict(list)

111

t = threading.Thread(target=run, args=(i, d))

112

results.append(d)

113

threads.append(t)

114

t.start()

115

for t in threads:

116

t.join()

117

result = defaultdict(list)

118

for d in results:

119

for k, v in d.items():

120

result[k].extend(v)

121

return result

122

123

def get_subscribers_json(packages, subscribers_json):

124

if subscribers_json is None:

125

j = urlopen("http://people.canonical.com/~ubuntu-archive/package-team-mapping.json")

126

else:

127

j = open(subscribers_json, 'rb')

128

with j:

129

team_to_packages = json.loads(j.read().decode('utf-8'))

130

package_to_teams = {}

131

for team, packages in team_to_packages.items():

132

for package in packages:

133

package_to_teams.setdefault(package, []).append(team)

134

return package_to_teams

135

136

@attr.s

137

class ArchRegression:

138

arch = attr.ib(default=None)

139

log_link = attr.ib(default=None)

140

hist_link = attr.ib(default=None)

141

142

143

@attr.s

144

class Regression:

145

blocking = attr.ib(default=None) # source package name blocked

146

package = attr.ib(default=None) # source package name doing the blocking

147

version = attr.ib(default=None) # version that regressed

148

arches = attr.ib(default=None) # [ArchRegression]

149

150

@property

151

def package_version(self):

152

return self.package + '/' + self.version

153

154

@attr.s

155

class Problem:

156

kind = attr.ib(default=None) # 'blocked-in-proposed', 'regressing-other'

157

package_in_proposed = attr.ib(default=None) # name of package that's in proposed

158

regressing_package = attr.ib(default=None) # name of package regressing package_in_proposed, None if blocked-in-proposed

159

regressions = attr.ib(default=None) # [Regression]

160

waiting = attr.ib(default=None) # [(source_package_name, arches)]

161

data = attr.ib(default=None) # data for package_in_proposed

162

unsatdepends = attr.ib(default=None) # [string]

163

164

_age = attr.ib(default=None)

165

166

@property

167

def age(self):

168

if self._age is not None:

169

return self._age

170

else:

171

return self.data["policy_info"]["age"]["current-age"]

172

@age.setter

173

def age(self, val):

174

self._age = val

175

176

177

def main():

178

parser = argparse.ArgumentParser()

179

parser.add_argument('--ppa', action='store')

180

parser.add_argument('--components', action='store', default="main,restricted")

181

parser.add_argument('--subscribers-from-lp', action='store_true')

182

parser.add_argument('--subscribers-json', action='store')

183

parser.add_argument('--true-ages', action='store_true')

184

parser.add_argument('--excuses-yaml', action='store')

185

parser.add_argument('output')

186

args = parser.parse_args()

187

188

components = args.components.split(',')

189

190

print("fetching yaml")

191

if args.excuses_yaml:

192

yaml_text = open(args.excuses_yaml).read()

193

else:

194

yaml_text = urlopen("https://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.yaml").read()

195

print("parsing yaml")

196

# The CSafeLoader is ten times faster than the regular one

197

excuses = yaml.load(yaml_text, Loader=yaml.CSafeLoader)

198

199

print("pre-processing packages")

200

in_proposed_packages = {}

201

for item in excuses["sources"]:

202

source_package_name = item['item-name']

203

# Missing component means main

204

if item.get('component', 'main') not in components:

205

continue

206

prob = Problem(kind='package-in-proposed', data=item, package_in_proposed=source_package_name)

207

in_proposed_packages[source_package_name] = prob

208

prob.regressions = []

209

prob.waiting = []

210

if 'autopkgtest' in item['reason']:

211

for package, results in sorted(item['policy_info']['autopkgtest'].items()):

212

regr_arches = []

213

wait_arches = []

214

for arch, result in sorted(results.items()):

215

outcome, log, history, wtf1, wtf2 = result

216

if outcome == "REGRESSION":

217

regr_arches.append(ArchRegression(arch=arch, log_link=log, hist_link=history))

218

if outcome == "RUNNING":

219

wait_arches.append(arch)

220

if regr_arches:

221

p, v = package.split('/')

222

regr = Regression(package=p, version=v, blocking=source_package_name)

223

regr.arches = regr_arches

224

prob.regressions.append(regr)

225

if wait_arches:

226

prob.waiting.append((package + ": " + ", ".join(wait_arches)))

227

if 'dependencies' in item and 'unsatisfiable-dependencies' in item['dependencies']:

228

unsatd = defaultdict(list)

229

for arch, packages in item['dependencies']['unsatisfiable-dependencies'].items():

230

for p in packages:

231

unsatd[p].append(arch)

232

prob.unsatdepends = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatd.items())]

233

234

package_to_problems = defaultdict(list)

235

236

for problem in in_proposed_packages.values():

237

package_to_problems[problem.package_in_proposed].append(problem)

238

for regression in problem.regressions:

239

if regression.blocking not in in_proposed_packages:

240

continue

241

if regression.blocking == regression.package:

242

continue

243

package_to_problems[regression.package].append(Problem(

244

kind='regressing-other', package_in_proposed=regression.blocking,

245

regressing_package=regression.package,

246

regressions=[regression],

247

data=in_proposed_packages[regression.blocking].data))

248

249

if args.true_ages:

250

true_ages = get_true_ages_in_proposed(set(package_to_problems), 10)

251

for package, true_age in true_ages.items():

252

for problem in package_to_problems[package]:

253

problem.age = true_age

254

255

print("getting subscribers")

256

if args.subscribers_from_lp:

257

subscribers = get_subscribers_lp(set(package_to_problems), 10)

258

for p in set(package_to_problems):

259

if p not in subscribers:

260

subscribers[p] = ['unsubscribed']

261

else:

262

subscribers = get_subscribers_json(set(package_to_problems), args.subscribers_json)

263

for p in set(package_to_problems):

264

if p not in subscribers:

265

subscribers[p] = ['unknown']

266

267

all_teams = set()

268

team_to_problems = defaultdict(list)

269

for package, teams in subscribers.items():

270

all_teams |= set(teams)

271

for team in teams:

272

team_to_problems[team].extend(package_to_problems[package])

273

274

for packages in team_to_problems.values():

275

packages.sort(key=lambda prob: (-prob.age, prob.regressing_package or prob.package_in_proposed))

276

277

print("rendering")

278

t = env.get_template('team-report.html')

279

with open(args.output, 'w', encoding='utf-8') as fp:

280

fp.write(t.render(

281

all_teams=all_teams,

282

team_to_problems=team_to_problems,

283

now=excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S")))

284

285

if __name__ == '__main__':

286

main()