~ubuntu-archive/ubuntu-archive-scripts/trunk : revision 324

1

#!/usr/bin/python3

2

3

# This script highlights packages that are being kept out of Debian

4

# testing by release critical bugs whose removal from the Ubuntu devel

5

# series would aid the migration of other packages out of proposed.

6

#

7

# The packages that are being kept out of Debian testing by release

8

# critical bugs can be found fairly easily by looking at the output

9

# from Debian's britney runs. We do this first (and call these

10

# "rc-gone packages").

11

#

12

# Such packages can inhibit the migration of other packages in two

13

# ways:

14

#

15

# 1) autopkgtest regressions

16

# 2) by becoming uninstallable

17

#

18

# The first is fairly easy to find: scan through Ubuntu's excuses.yaml

19

# and look for the rc-gone package in the list of autopkgtest

20

# regressions for any package.

21

#

22

# The second is a bit more mind-bending to detect. If the package is

23

# caught up in a transition, a rebuild will be attempted as a matter

24

# of course, and if this succeeds and passes its autopkgtests then

25

# removing the package will not aid proposed migration. So we look for

26

# rc-gone packages in proposed that are missing builds or failing

27

# autopkgtests. For such source packages, we see if any of their

28

# binary packages are reported as being made uninstallable by the

29

# migration of any source package in proposed. If so, removing the

30

# rc-gone package will help proposed migration.

31

32

import argparse

33

import os

34

import shlex

35

import subprocess

36

37

import attr

38

from jinja2 import Environment, FileSystemLoader

39

import yaml

40

41

env = Environment(

42

loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),

43

autoescape=True,

44

extensions=['jinja2.ext.i18n'],

45

)

46

env.install_null_translations(True)

47

48

def parse_args():

49

parser = argparse.ArgumentParser()

50

parser.add_argument('--ubuntu-excuses', action='store')

51

parser.add_argument('--ubuntu-update_output', action='store')

52

parser.add_argument('--debian-excuses', action='store')

53

parser.add_argument('--output', action='store')

54

return parser.parse_args()

55

56

args = parse_args()

57

58

def run_output(*cmd, **extra):

59

encoding = extra.pop('encoding', 'ascii')

60

kw = dict(check=True, stdout=subprocess.PIPE)

61

kw.update(extra)

62

cp = subprocess.run(cmd, **kw)

63

return cp.stdout.decode(encoding).strip()

64

65

all_arches = set()

66

67

def extract_bin_pkg_arch_to_blocked_src_pkgs(output_fp):

68

# Extract a mapping from binary package name / architectures to

69

# source package the migration of which would make that package

70

# uninstallable (and for convenience, return all architectures

71

# used as keys -- usually there will only be one or two).

72

73

# We're looking for sequences of lines like this:

74

75

# skipped: camlp5 (0, 3, 57)

76

# got: 13+0: a-1:a-0:a-0:i-6:p-0:s-6

77

# * s390x: hol-light, libaac-tactics-ocaml-dev, libcoq-ocaml-dev, libledit-ocaml-dev, ocaml-ulex08

78

79

# (Britney tries to migrate batches of packages but it always

80

# tries each package on its own as well).

81

82

# For each potential migration, britney checks architectures in

83

# sequence and stops when it finds one that regresses (or proceeds

84

# with the migration if it doesn't find one). This means that we

85

# can miss blocking packages here --- for example if migrating $src

86

# would make say $binpkg/amd64 uninstallable, but britney happens

87

# to check arm64 -- where $binpkg does not exist -- and there are

88

# regressions there, we will never find out about the problem

89

# $binpkg causes. This isn't really too bad because clearly in

90

# this case the migration of $src is blocked by other things that

91

# need to be resolved, but it does mean that packages might appear

92

# and disapper from the report depending on the order that britney

93

# checks architectures in (which is not consistent from run to

94

# run). C'est la vie.

95

96

bin_pkg_arch_to_blocked_src_pkgs = {}

97

srcpkg = None

98

arch_prefix = "Arch order is: "

99

for line in output_fp:

100

line = line.strip()

101

if line.startswith(arch_prefix):

102

all_arches.update(line[len(arch_prefix):].split(', '))

103

parts = line.split(None, 2)

104

if len(parts) >= 2:

105

if parts[0] in {"Trying", "trying"}:

106

srcpkg = None

107

if parts[0] == 'skipped:':

108

srcpkg = None

109

# If parts[2] is '(' then this line is about trying to

110

# migrate a single package, which is what we are

111

# looking for.

112

if parts[2].startswith('('):

113

srcpkg = parts[1]

114

if srcpkg is not None and srcpkg[0] != '-' and parts[0] == '*':

115

# parts[1] is "${arch}:"

116

# parts[2:] is a comma+space separated list of binary package names.

117

arch = parts[1][:-1]

118

for binpkg in parts[2].split(', '):

119

bin_pkg_arch_to_blocked_src_pkgs.setdefault(

120

(binpkg, arch), set()).add(srcpkg)

121

return bin_pkg_arch_to_blocked_src_pkgs

122

123

124

def chdist_grep_dctrl_packages(arch, *args):

125

return run_output(

126

"chdist", "grep-dctrl-packages", "{}-{}".format(series, arch),

127

"-nsPackage", *args, check=False).splitlines()

128

129

130

def chdist_grep_dctrl_sources(arch, *args):

131

return run_output(

132

"chdist", "grep-dctrl-sources", "{}-{}".format(series, arch),

133

"-nsPackage", *args, check=False).splitlines()

134

135

136

@attr.s

137

class RCGone:

138

source_package_name = attr.ib()

139

bugs = attr.ib()

140

block_by_regression = attr.ib(default=attr.Factory(set))

141

block_by_uninstallability = attr.ib(default=attr.Factory(set))

142

suites = attr.ib(default=attr.Factory(set))

143

_rdeps_lines = attr.ib(default=None)

144

_binpkgs = attr.ib(default=None)

145

146

def binary_pkgs(self):

147

if self._binpkgs is None:

148

self._binpkgs = set()

149

for arch in all_arches:

150

arch_binpkgs = chdist_grep_dctrl_packages(

151

arch, "-wS", self.source_package_name)

152

self._binpkgs.update({(binpkg, arch) for binpkg in arch_binpkgs})

153

return self._binpkgs

154

155

def reverse_depends(self):

156

if self._rdeps_lines is None:

157

# These are maps rdep -> binpkg -> arches

158

reverse_recommends = {}

159

reverse_depends = {}

160

# This just maps rbdep -> binpkgs

161

reverse_build_depends = {}

162

for binpkg, arch in self.binary_pkgs():

163

for rec in chdist_grep_dctrl_packages(

164

arch, "-wFRecommends", binpkg):

165

if (rec, arch) in self.binary_pkgs():

166

continue

167

reverse_recommends.setdefault(rec, {}).setdefault(binpkg, set()).add(arch)

168

for dep in chdist_grep_dctrl_packages(

169

arch, "-wFDepends", binpkg):

170

if (dep, arch) in self.binary_pkgs():

171

continue

172

reverse_depends.setdefault(dep, {}).setdefault(binpkg, set()).add(arch)

173

for bdeb in chdist_grep_dctrl_sources(

174

arch, "-wFBuild-Depends", binpkg, "--or", "-wFBuild-Depends-Indep", binpkg):

175

reverse_build_depends.setdefault(bdeb, set()).add(binpkg)

176

self._rdeps_lines = []

177

if reverse_depends:

178

self._rdeps_lines.append("Reverse-Depends")

179

for rdep in sorted(reverse_depends):

180

for binpkg in sorted(reverse_depends[rdep]):

181

if reverse_depends[rdep][binpkg] == all_arches:

182

arches = "all architectures"

183

else:

184

arches = ", ".join(reverse_depends[rdep][binpkg])

185

self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg, "on", arches]))

186

if reverse_recommends:

187

self._rdeps_lines.append("Reverse-Recommends")

188

for rdep in sorted(reverse_recommends):

189

for binpkg in sorted(reverse_recommends[rdep]):

190

if reverse_recommends[rdep][binpkg] == all_arches:

191

arches = "all architectures"

192

else:

193

arches = ", ".join(reverse_recommends[rdep][binpkg])

194

self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg, "on", arches]))

195

if reverse_build_depends:

196

self._rdeps_lines.append("Reverse-Build-Depends")

197

for rdep in sorted(reverse_build_depends):

198

for binpkg in sorted(reverse_build_depends[rdep]):

199

self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg]))

200

return self._rdeps_lines

201

202

@property

203

def comment(self):

204

comment = "removed from testing (Debian bug"

205

if len(self.bugs) > 1:

206

comment += "s"

207

comment += " " + ", ".join('#' + b for b in self.bugs) + ")"

208

if self.block_by_regression:

209

comment += ', blocks {} by autopkgtest regression'.format(', '.join(sorted(self.block_by_regression)))

210

if self.block_by_uninstallability:

211

comment += ', blocks {} by uninstallability'.format(', '.join(sorted(self.block_by_uninstallability)))

212

return comment

213

214

@property

215

def rdeps_text_short(self):

216

return "\n".join(self.reverse_depends()[:10])

217

218

@property

219

def rdeps_text_more(self):

220

return "\n".join(self.reverse_depends()[10:])

221

222

@property

223

def removal_commands(self):

224

suites = self.suites

225

if not suites:

226

suites = [series]

227

cmds = []

228

for suite in suites:

229

cmd = " ".join(["remove-package", "-s", suite, "-y", "-m", shlex.quote(self.comment), self.source_package_name])

230

if self.reverse_depends():

231

cmd = '#' + cmd

232

cmds.append(cmd)

233

return cmds

234

235

@property

236

def css_class(self):

237

if self.reverse_depends():

238

return 'removal-not-ok'

239

else:

240

return 'removal-ok'

241

242

243

if 'SERIES' in os.environ:

244

series = os.environ['SERIES']

245

else:

246

series = run_output('distro-info', '-d')

247

248

249

def main():

250

print("loading data")

251

if args.ubuntu_excuses.endswith('.xz'):

252

import lzma

253

excuses_opener = lzma.open

254

else:

255

excuses_opener = open

256

with excuses_opener(args.ubuntu_excuses) as fp:

257

ubuntu_excuses = yaml.load(fp, Loader=yaml.CSafeLoader)

258

with open(args.ubuntu_update_output) as fp:

259

bin_pkg_arch_to_blocked_src_pkgs = extract_bin_pkg_arch_to_blocked_src_pkgs(fp)

260

with open(args.debian_excuses) as fp:

261

debian_excuses = yaml.load(fp, Loader=yaml.CSafeLoader)

262

263

print("finding rcgone packages")

264

rc_gones = {}

265

266

for source in debian_excuses['sources']:

267

if source['old-version'] == '-':

268

info = source['policy_info']

269

rc_bugs = info.get('rc-bugs', {})

270

if rc_bugs.get('verdict') == "REJECTED_PERMANENTLY":

271

bugs = []

272

for k in 'shared-bugs', 'unique-source-bugs', 'unique-target-bugs':

273

bugs.extend(rc_bugs[k])

274

rc_gones[source['item-name']] = RCGone(

275

source_package_name=source['item-name'],

276

bugs=bugs)

277

in_proposed_by_autopkgtest_or_missing_binaries = set()

278

print("checking autopkgtests")

279

for source in ubuntu_excuses['sources']:

280

item = source['item-name']

281

if 'autopkgtest' in source['reason']:

282

in_proposed_by_autopkgtest_or_missing_binaries.add(item)

283

for package, results in sorted(source['policy_info']['autopkgtest'].items()):

284

if '/' not in package:

285

# This only happens when all tests are still running

286

continue

287

package, version = package.split('/')

288

if package not in rc_gones or '-0ubuntu' in version:

289

continue

290

for arch, result in sorted(results.items()):

291

outcome, log, history, wtf1, wtf2 = result

292

if outcome == "REGRESSION" and package != item:

293

rc_gones[package].block_by_regression.add(item)

294

break

295

if 'missing-builds' in source and '-0ubuntu' not in source['new-version']:

296

in_proposed_by_autopkgtest_or_missing_binaries.add(item)

297

if item in rc_gones:

298

if source['new-version'] != '-':

299

rc_gones[item].suites.add(series+"-proposed")

300

if source['old-version'] != '-':

301

rc_gones[item].suites.add(series)

302

print("checking uninstallability")

303

for rc_gone in rc_gones.values():

304

if rc_gone.source_package_name not in in_proposed_by_autopkgtest_or_missing_binaries:

305

continue

306

for bin_pkg, arch in set(rc_gone.binary_pkgs()):

307

rc_gone.block_by_uninstallability.update(bin_pkg_arch_to_blocked_src_pkgs.get((bin_pkg, arch), set()))

308

print("finding reverse-deps")

309

packages = []

310

for _, rc_gone in sorted(rc_gones.items()):

311

if not rc_gone.block_by_regression and not rc_gone.block_by_uninstallability:

312

continue

313

rc_gone.reverse_depends()

314

packages.append(rc_gone)

315

316

print("rendering")

317

t = env.get_template('rcbuggy-problem-packages.html')

318

with open(args.output, 'w', encoding='utf-8') as fp:

319

fp.write(t.render(

320

packages=packages,

321

now=ubuntu_excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S")))

322

323

324

if __name__ == '__main__':

325

main()