« back to all changes in this revision
Viewing changes to sru-release
- browse files at revision 1505
- compare with another revision
- download diff
- download tarball
- view history from revision 1505
- Committer: Steve Langasek
- Date: 2021-08-31 00:41:56 UTC
- mfrom: (1425.1.1 ubuntu-archive-tools.bzrinfo.0.repo-cache.bzr-redirect)
- Revision ID: vorlon@debian.org-20210831004156-j48n1sws63p3j504
Merge lp:~rbalint/ubuntu-archive-tools/bzr-to-git
- files added:
- THIS_REPOSITORY_HAS_BEEN_MOVED_TO_GIT
- files removed:
- .bzrignore
- edit_acl.py
added
removed
sru-release
1
#!/usr/bin/python3
2
3
# Copyright (C) 2011, 2012 Canonical Ltd.
4
# Author: Martin Pitt <martin.pitt@canonical.com>
5
6
# This program is free software: you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; version 3 of the License.
9
#
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18
'''Release a proposed stable release update.
19
20
Copy packages from -proposed to -updates, and optionally to -security and the
21
development release.
22
23
USAGE:
24
sru-release [-s] [-d] <release> <package> [<package> ...]
25
'''
26
27
from __future__ import print_function
28
29
from collections import defaultdict
30
from functools import partial
31
import datetime
32
import optparse
33
import os
34
import subprocess
35
import sys
36
import time
37
import unittest
38
39
from six.moves.urllib.request import urlopen
40
from io import TextIOWrapper
41
42
from launchpadlib.launchpad import Launchpad
43
44
from kernel_series import KernelSeries
45
46
47
# Each entry in this list is a list of source packages that are known
48
# to have inter-dependencies and must be released simultaneously.
49
# If possible, each list should be ordered such that earlier
50
# entries could be released slightly before subsequent entries.
51
RELEASE_TOGETHER_PACKAGE_GROUPS = [
52
['linux-hwe', 'linux-signed-hwe', 'linux-meta-hwe'],
53
['linux', 'linux-signed', 'linux-meta'],
54
['grub2-unsigned', 'grub2-signed'],
55
['s390-tools', 's390-tools-signed'],
56
['shim', 'shim-signed'],
57
]
58
59
MISSING_PACKAGES_FROM_GROUP = (
60
"The set of packages requested for release are listed as dangerous \n"
61
"to release without also releasing the following at the same time:\n"
62
" {missing}\n\n"
63
"For more information, see:\n"
64
" https://lists.ubuntu.com/archives/ubuntu-devel/2018-June/040380.html\n\n"
65
"To ignore this message, pass '--skip-package-group-check'.")
66
67
BZR_HINT_BRANCH = "lp:~ubuntu-sru/britney/hints-ubuntu-%s"
68
69
70
def check_package_sets(packages):
71
"""Return a re-ordered list of packages respecting the PACKAGE_SETS
72
defined above. If any packages are missing, raise error."""
73
74
# pkg2group is a dict where each key is a pkg in a group and value is the
75
# complete group.
76
pkg2group = {}
77
for pgroup in RELEASE_TOGETHER_PACKAGE_GROUPS:
78
for pkg in pgroup:
79
if pkg in pkg2group:
80
raise RuntimeError(
81
"Overlapping package groups. '%s' is in '%s' and '%s'." %
82
(pkg, pgroup, pkg2group[pkg]))
83
pkg2group[pkg] = pgroup
84
85
seen = set()
86
new_pkgs = []
87
for pkg in packages:
88
if pkg not in pkg2group:
89
add = [pkg]
90
else:
91
add = list(pkg2group[pkg])
92
new_pkgs.extend([a for a in add if a not in seen])
93
seen.update(add)
94
95
orig = set(packages)
96
new = set(new_pkgs)
97
if orig != new:
98
raise ValueError(
99
MISSING_PACKAGES_FROM_GROUP.format(
100
missing=' '.join(new.difference(orig))))
101
return new_pkgs
102
103
104
class CheckPackageSets(unittest.TestCase):
105
def test_expected_linux_order_fixed(self):
106
self.assertEqual(
107
['pkg1', 'linux', 'linux-signed', 'linux-meta', 'pkg2'],
108
check_package_sets(['pkg1', 'linux-meta', 'linux', 'linux-signed', 'pkg2']))
109
110
def test_raises_value_error_on_missing(self):
111
self.assertRaises(
112
ValueError, check_package_sets, ['pkg1', 'linux'])
113
114
def test_single_item_with_missing(self):
115
self.assertRaises(
116
ValueError, check_package_sets, ['linux'])
117
118
def test_single_item_without_missing(self):
119
self.assertEqual(
120
check_package_sets(['pkg1']), ['pkg1'])
121
122
def test_multiple_package_groups(self):
123
"""Just make sure that having multiple groups listed still errors."""
124
self.assertRaises(
125
ValueError, check_package_sets, ['pkg1', 'linux', 'grub2'])
126
127
128
def match_srubugs(options, changesfileurl):
129
'''match between bugs with verification- tag and bugs in changesfile'''
130
131
bugs = []
132
133
if changesfileurl is None:
134
return bugs
135
136
# Load changesfile
137
changelog = TextIOWrapper(urlopen(changesfileurl), encoding='utf-8')
138
bugnums = []
139
for l in changelog:
140
if l.startswith('Launchpad-Bugs-Fixed: '):
141
bugnums = l.split()[1:]
142
break
143
144
for b in bugnums:
145
if b in options.exclude_bug:
146
continue
147
try:
148
bugs.append(launchpad.bugs[int(b)])
149
except:
150
print('%s: bug %s does not exist or is not accessible' %
151
(changesfileurl, b))
152
153
return bugs
154
155
156
def update_sru_bug(bug, pkg):
157
'''Unsubscribe SRU team and comment on bug re: how to report regressions'''
158
m_subjects = [m.subject for m in bug.messages]
159
if 'Update Released' in m_subjects:
160
print('LP: #%s was not commented on' % bug.id)
161
return
162
sru_team = launchpad.people['ubuntu-sru']
163
bug.unsubscribe(person=sru_team)
164
text = ("The verification of the Stable Release Update for %s has "
165
"completed successfully and the package is now being released "
166
"to -updates. Subsequently, the Ubuntu Stable Release Updates "
167
"Team is being unsubscribed and will not receive messages "
168
"about this bug report. In the event that you encounter "
169
"a regression using the package from -updates please report "
170
"a new bug using ubuntu-bug and tag the bug report "
171
"regression-update so we can easily find any regressions." % pkg)
172
bug.newMessage(subject="Update Released", content=text)
173
bug.lp_save()
174
175
176
def get_versions(options, sourcename):
177
'''Get current package versions.
178
179
If options.pattern is True, return all versions for package names
180
matching options.pattern.
181
If options.pattern is False, only return one result.
182
183
Return map pkgname -> {'release': version, 'updates': version,
184
'proposed': version, 'changesfile': url_of_proposed_changes,
185
'published': proposed_date}
186
'''
187
versions = defaultdict(dict)
188
if src_archive.reference == 'ubuntu':
189
pocket = 'Proposed'
190
else:
191
pocket = 'Release'
192
193
matches = src_archive.getPublishedSources(
194
source_name=sourcename, exact_match=not options.pattern,
195
status='Published', pocket=pocket, distro_series=series)
196
for match in matches:
197
# versions in all pockets
198
for pub in src_archive.getPublishedSources(
199
source_name=match.source_package_name, exact_match=True,
200
status='Published', distro_series=series):
201
key = pub.pocket.lower()
202
# special case for ppas, which don't have pockets but need
203
# to be treated as -proposed
204
if pocket == 'Release' and key == 'release':
205
key = 'proposed'
206
versions[pub.source_package_name][key] = (
207
pub.source_package_version)
208
if pocket in pub.pocket:
209
versions[pub.source_package_name]['changesfile'] = (
210
pub.changesFileUrl())
211
# When the destination archive differs from the source scan that too.
212
if dst_archive != src_archive:
213
for pub in dst_archive.getPublishedSources(
214
source_name=match.source_package_name, exact_match=True,
215
status='Published', distro_series=series):
216
key = 'security' # pub.pocket.lower()
217
versions[pub.source_package_name][key] = (
218
pub.source_package_version)
219
if pocket in pub.pocket:
220
versions[pub.source_package_name]['changesfile'] = (
221
pub.changesFileUrl())
222
# devel version
223
if devel_series:
224
for pub in src_archive.getPublishedSources(
225
source_name=match.source_package_name, exact_match=True,
226
status='Published', distro_series=devel_series):
227
if pub.pocket in ('Release', 'Proposed'):
228
versions[pub.source_package_name]['devel'] = (
229
pub.source_package_version)
230
else:
231
versions[match.source_package_name]['devel'] = None
232
233
return versions
234
235
236
def release_packages(options, packages):
237
'''Release the packages listed in the packages argument.'''
238
239
pkg_versions_map = {}
240
# Dictionary of packages and their versions that need copying by britney.
241
# Those packages have unblock hints added.
242
packages_to_britney = {}
243
244
for package in packages:
245
pkg_versions_map[package] = get_versions(options, package)
246
if not pkg_versions_map[package]:
247
message = ('ERROR: No such package, ' + package + ', in '
248
'-proposed, aborting\n')
249
sys.stderr.write(message)
250
sys.exit(1)
251
252
# If any bug is tagged 'block-proposed' prevent releasing the update.
253
for pkg_versions in pkg_versions_map.values():
254
for pkg, versions in pkg_versions.items():
255
sru_bugs = match_srubugs(options, versions['changesfile'])
256
block_tag = 'block-proposed-%s' % release
257
failed_tag = 'verification-failed-%s' % release
258
message = None
259
for sru_bug in sru_bugs:
260
if block_tag in sru_bug.tags:
261
message = ('ERROR: not releasing ' + pkg + ' as SRU '
262
'bug ' + str(sru_bug.id) + ' is tagged '
263
+ block_tag + '\n')
264
elif failed_tag in sru_bug.tags:
265
message = ('ERROR: not releasing ' + pkg + ' as SRU '
266
'bug ' + str(sru_bug.id) + ' is tagged '
267
+ failed_tag + '\n')
268
if message:
269
sys.stderr.write(message)
270
sys.exit(1)
271
272
for pkg, versions in pkg_versions_map[package].items():
273
print('--- Releasing %s ---' % pkg)
274
print('Proposed: %s' % versions['proposed'])
275
if 'security' in versions:
276
print('Security: %s' % versions['security'])
277
if 'updates' in versions:
278
print('Updates: %s' % versions['updates'])
279
else:
280
print('Release: %s' % versions.get('release'))
281
if options.devel and 'devel' in versions:
282
print('Devel: %s' % versions['devel'])
283
284
copy = partial(
285
dst_archive.copyPackage, from_archive=src_archive,
286
include_binaries=True, source_name=pkg,
287
version=versions['proposed'], auto_approve=True)
288
289
if options.devel and not options.britney:
290
if ('devel' not in versions or
291
versions['devel'] in (
292
versions.get('updates', 'notexisting'),
293
versions['release'])):
294
if not options.no_act:
295
copy(to_pocket='Proposed', to_series=devel_series.name)
296
print('Version in %s matches development series, '
297
'copied to %s-proposed' %
298
(release, devel_series.name))
299
else:
300
print('ERROR: Version in %s does not match development '
301
'series, not copying' % release)
302
303
if options.no_act:
304
if options.release:
305
print('Would copy to %s' % release)
306
else:
307
print('Would copy to %s-updates' % release)
308
else:
309
if options.release:
310
# -proposed -> release
311
copy(to_pocket='Release', to_series=release)
312
print('Copied to %s' % release)
313
else:
314
# -proposed -> -updates
315
if (package != 'linux' and
316
not package.startswith('linux-') and
317
not options.security):
318
if options.britney:
319
# We can opt in to use britney for the package copy
320
# instead of doing direct pocket copies.
321
packages_to_britney[pkg] = versions['proposed']
322
else:
323
copy(to_pocket='Updates', to_series=release,
324
phased_update_percentage=options.percentage)
325
print('Copied to %s-updates' % release)
326
else:
327
copy(to_pocket='Updates', to_series=release)
328
print('Copied to %s-updates' % release)
329
330
# -proposed -> -security
331
if options.security:
332
if options.no_act:
333
print('Would copy to %s-security' % release)
334
else:
335
copy(to_pocket='Security', to_series=release)
336
print('Copied to %s-security' % release)
337
338
# Write hints for britney to copy the selected packages
339
if options.britney and packages_to_britney:
340
release_package_via_britney(options, packages_to_britney)
341
# If everything went well, update the bugs
342
if not options.no_bugs:
343
for pkg_versions in pkg_versions_map.values():
344
for pkg, versions in pkg_versions.items():
345
sru_bugs = match_srubugs(options, versions['changesfile'])
346
tag = 'verification-needed-%s' % release
347
for sru_bug in sru_bugs:
348
if tag not in sru_bug.tags:
349
update_sru_bug(sru_bug, pkg)
350
351
352
def release_package_via_britney(options, packages):
353
'''Release selected packages via britney unblock hints.'''
354
355
hints_path = os.path.join(options.cache, 'hints-ubuntu-%s' % release)
356
hints_file = os.path.join(hints_path, 'sru-release')
357
# Checkout the hints branch
358
if not os.path.exists(hints_path):
359
cmd = ['bzr', 'checkout', '--lightweight',
360
BZR_HINT_BRANCH % release, hints_path]
361
else:
362
cmd = ['bzr', 'update', hints_path]
363
try:
364
subprocess.check_call(cmd)
365
except subprocess.CalledProcessError:
366
sys.stderr.write("Failed bzr %s for the hints branch at %s\n" %
367
(cmd[1], hints_path))
368
sys.exit(1)
369
# Create the hint with a timestamp comment
370
timestamp = time.time() # In python2 we can't use datetime.timestamp()
371
date = datetime.datetime.now().ctime()
372
unblock_string = '# %s %s\n' % (timestamp, date)
373
unblock_string += ''.join(['unblock %s/%s\n' % (pkg, ver)
374
for pkg, ver in packages.items()])
375
unblock_string += '\n'
376
# Update and commit the hint
377
with open(hints_file, 'a+') as f:
378
f.write(unblock_string)
379
cmd = ['bzr', 'commit', '-m', 'sru-release %s %s' %
380
(release, ' '.join(packages.keys()))]
381
try:
382
subprocess.check_call(cmd)
383
except subprocess.CalledProcessError:
384
sys.stderr.write('Failed to bzr commit to the hints file %s\n'
385
'Please investigate the local hint branch and '
386
'commit the required unblock entries as otherwise '
387
'your changes will be lost.\n' %
388
hints_file)
389
sys.exit(1)
390
print('Added hints for promotion in release %s of packages %s' %
391
(release, ' '.join(packages.keys())))
392
393
394
if __name__ == '__main__':
395
if len(sys.argv) > 1 and sys.argv[1] == "run-tests":
396
sys.exit(unittest.main(argv=[sys.argv[0]] + sys.argv[2:]))
397
398
parser = optparse.OptionParser(
399
usage='usage: %prog [options] <release> <package> [<package> ...]')
400
401
parser.add_option(
402
'-l', '--launchpad', dest='launchpad_instance', default='production')
403
parser.add_option(
404
'--security', action='store_true', default=False,
405
help='Additionally copy to -security pocket')
406
parser.add_option(
407
'-d', '--devel', action='store_true', default=False,
408
help='Additionally copy to development release (only works if that '
409
'has the same version as <release>)')
410
parser.add_option(
411
'-r', '--release', action='store_true', default=False,
412
help='Copy to release pocket instead of -updates (useful for staging '
413
'uploads in development release)')
414
parser.add_option(
415
"-z", "--percentage", type="int", default=10,
416
metavar="PERCENTAGE", help="set phased update percentage")
417
parser.add_option(
418
'-n', '--no-act', action='store_true', default=False,
419
help='Only perform checks, but do not actually copy packages')
420
parser.add_option(
421
'-p', '--pattern', action='store_true', default=False,
422
help='Treat package names as patterns, not exact matches')
423
parser.add_option(
424
'--no-bugs', action='store_true', default=False,
425
help='Do not act on any bugs (helpful to avoid races).')
426
parser.add_option(
427
'--exclude-bug', action='append', default=[], metavar='BUG',
428
help='Do not update BUG.')
429
parser.add_option(
430
'-E', '--esm', action='store_true', default=False,
431
help='Copy from the kernel ESM proposed PPA to the ESM publication PPA')
432
parser.add_option(
433
'--skip-package-group-check', action='store_true', default=False,
434
help=('Skip the package set checks that require some packages '
435
'be released together'))
436
parser.add_option(
437
'--britney', action='store_true', default=False,
438
help='Use britney for copying the packages over to -updates (only '
439
'works for regular package releases into updates)')
440
parser.add_option(
441
'-C', '--cache', default='~/.cache/sru-release',
442
help='Cache directory to be used for the britney hints checkout')
443
444
options, args = parser.parse_args()
445
446
if len(args) < 2:
447
parser.error(
448
'You must specify a release and source package(s), see --help')
449
450
if options.release and (options.security or options.devel):
451
parser.error('-r and -s/-d are mutually exclusive, see --help')
452
453
release = args.pop(0)
454
packages = args
455
456
# XXX: we only want to instantiate KernelSeries if we suspect this is
457
# a kernel package, this is necessarily dirty, dirty, dirty.
458
kernel_checks = False
459
for package in packages:
460
if package.startswith('linux-') or package == 'linux':
461
kernel_checks = True
462
463
if not options.skip_package_group_check:
464
try:
465
packages = check_package_sets(packages)
466
except ValueError as e:
467
sys.stderr.write(e.args[0] + '\n')
468
sys.exit(1)
469
470
options.cache = os.path.expanduser(options.cache)
471
if not os.path.isdir(options.cache):
472
if os.path.exists(options.cache):
473
print('Cache path %s already exists and is not a directory.'
474
% options.cache)
475
sys.exit(1)
476
os.makedirs(options.cache)
477
478
launchpad = Launchpad.login_with(
479
'ubuntu-archive-tools', options.launchpad_instance, version='devel')
480
ubuntu = launchpad.distributions['ubuntu']
481
series = ubuntu.getSeries(name_or_version=release)
482
devel_series = ubuntu.current_series
483
if not devel_series:
484
sys.stderr.write(
485
'WARNING: No current development series, -d will not work\n')
486
devel_series = None
487
488
ks_source = None
489
if kernel_checks:
490
kernel_series = KernelSeries()
491
492
# See if we have a kernel-series record for this package. If we do
493
# then we are going to pivot to the routing therein.
494
ks_series = kernel_series.lookup_series(codename=release)
495
for ks_source_find in ks_series.sources:
496
for ks_package in ks_source_find.packages:
497
if ks_package.name == packages[0]:
498
ks_source = ks_source_find
499
break
500
501
# First confirm everything in this set we are attempting to release
502
# are indeed listed as valid for this kernel.
503
if ks_source is not None:
504
for package in packages:
505
if ks_source.lookup_package(package) is None:
506
sys.stderr.write(
507
'WARNING: {} not found in packages for kernel {}\n'.format(
508
package, ks_source.name))
509
510
if ks_source is None and release in ('precise', 'trusty'):
511
sys.stdout.write(
512
'Called for {}; assuming kernel ESM publication\n'.format(release))
513
options.esm = True
514
515
# If we found a KernelSeries entry this has accurate routing information
516
# attached use that.
517
if ks_source is not None:
518
src_archive_ref, src_archive_pocket = ks_source.routing.lookup_destination('proposed', primary=True)
519
src_archive = launchpad.archives.getByReference(
520
reference=src_archive_ref)
521
dst_archive_ref, dst_archive_pocket = ks_source.routing.lookup_destination('updates', primary=True)
522
if dst_archive_ref == src_archive_ref:
523
dst_archive = src_archive
524
else:
525
dst_archive = launchpad.archives.getByReference(
526
reference=dst_archive_ref)
527
528
# Announce any non-standard archive routing.
529
if src_archive_ref != 'ubuntu':
530
print("Src Archive: {}".format(src_archive_ref))
531
if dst_archive_ref != 'ubuntu':
532
print("Dst Archive: {}".format(dst_archive_ref))
533
# --security is meaningless for private PPA publishing (XXX: currently true)
534
options.security = False
535
options.release = True
536
537
elif options.esm:
538
# --security is meaningless for ESM everything is a security update.
539
options.security = False
540
options.release = True
541
src_archive = launchpad.archives.getByReference(
542
reference='~canonical-kernel-esm/ubuntu/proposed')
543
dst_archive = launchpad.archives.getByReference(
544
reference='~ubuntu-esm/ubuntu/esm')
545
else:
546
src_archive = dst_archive = ubuntu.getArchive(name='primary')
547
548
release_packages(options, packages)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1