1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
|
#! /usr/bin/python3
# Copyright (C) 2014 Canonical Ltd.
# Author: Colin Watson <cjwatson@ubuntu.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Requires germinate >= 2.18.
"""Copy a subset of one distribution into a derived distribution."""
import atexit
from collections import OrderedDict
from contextlib import closing, contextmanager
import io
import logging
from optparse import OptionParser, Values
import os
import shutil
import subprocess
import sys
import tempfile
import time
from urllib.request import urlopen
import apt_pkg
from dateutil import parser as dateutil_parser
from germinate.archive import TagFile
from germinate.germinator import Germinator
from germinate.seeds import Seed, SeedError, SeedStructure
from launchpadlib.errors import HTTPError
from launchpadlib.launchpad import Launchpad
import pytz
from ubuntutools.question import YesNoQuestion
import lputils
_bzr_cache_dir = None
@contextmanager
def open_url_as_text(url):
with closing(urlopen(url)) as raw:
with closing(io.BytesIO(raw.read())) as binary:
with closing(io.TextIOWrapper(binary)) as text:
yield text
class ManifestError(Exception):
pass
class TimeTravellingSeed(Seed):
def __init__(self, options, *args):
self.options = options
super(TimeTravellingSeed, self).__init__(*args, bzr=True)
def _bzr_revision_at_date(self, branch_url, date_obj):
"""Work out the bzr revision of a branch at a particular date.
Unfortunately, bzr's date: revisionspec is unhelpful for this, as it
asks for the first revision *after* the given date, and fails if the
given date is after the last revision on the branch. We could
probably do this properly with bzrlib, but life's too short and this
will do.
This assumes all sorts of things like the exact ordering of field
names in log output. Since bzr is no longer being heavily
developed, hopefully this won't be a problem until we can switch the
seeds to git ...
"""
command = ["bzr", "log", branch_url]
bzr_log = subprocess.Popen(
command, stdout=subprocess.PIPE, universal_newlines=True)
revno = None
for line in bzr_log.stdout:
line = line.rstrip("\n")
if line.startswith("revno: "):
revno = line[len("revno: "):].split(" ", 1)[0]
elif line.startswith("timestamp: "):
timestamp = dateutil_parser.parse(line[len("timestamp: "):])
if timestamp < date_obj:
break
else:
revno = None
bzr_log.stdout.close()
bzr_log.wait()
if revno is None:
raise SeedError("No revision found at %s in %s" % (
date_obj, branch_url))
return revno
def _open_seed(self, base, branch, name, bzr=False):
if not bzr:
raise Exception("Non-bzr-based time travel is not supported.")
global _bzr_cache_dir
if _bzr_cache_dir is None:
_bzr_cache_dir = tempfile.mkdtemp(prefix="derive-distribution-")
atexit.register(shutil.rmtree, _bzr_cache_dir, ignore_errors=True)
path = os.path.join(base, branch)
checkout = os.path.join(_bzr_cache_dir, branch)
if not os.path.isdir(checkout):
revno = self._bzr_revision_at_date(path, self.options.date)
logging.info("Checking out %s at r%s" % (path, revno))
command = [
"bzr", "checkout", "--lightweight", "-r%s" % revno, path,
checkout,
]
status = subprocess.call(command)
if status != 0:
raise SeedError(
"Command failed with exit status %d:\n '%s'" % (
status, " ".join(command)))
return open(os.path.join(checkout, name))
class TimeTravellingSeedStructure(SeedStructure):
def __init__(self, options, *args, **kwargs):
kwargs["bzr"] = True
self.options = options
super(TimeTravellingSeedStructure, self).__init__(*args, **kwargs)
def make_seed(self, bases, branches, name, bzr=False):
if not bzr:
raise Exception("Non-bzr-based time travel is not supported.")
return TimeTravellingSeed(self.options, bases, branches, name)
class TimeTravellingGerminator:
apt_mirror = "http://people.canonical.com/~ubuntu-archive/apt-mirror.cgi"
def __init__(self, options):
self.options = options
@property
def components(self):
return ["main", "restricted", "universe", "multiverse"]
@property
def mirror(self):
if self.options.date is not None:
timestamp = int(time.mktime(self.options.date.timetuple()))
return "%s/%d" % (self.apt_mirror, timestamp)
else:
return self.apt_mirror
def makeSeedStructures(self, suite, flavours, extra_packages):
series_name = suite.split("-")[0]
if self.options.seed_source is None:
seed_bases = None
else:
seed_bases = self.options.seed_source.split(",")
structures = {}
for flavour in flavours:
try:
structure = TimeTravellingSeedStructure(
self.options, "%s.%s" % (flavour, series_name),
seed_bases=seed_bases)
if len(structure):
extra_seed = []
for extra_package in extra_packages:
extra_seed.append(" * " + extra_package)
if extra_seed:
structure.add("extra-packages", extra_seed, "required")
# Work around inability to specify extra packages
# with no parent seeds.
structure._inherit["extra-packages"] = []
structures[flavour] = structure
# TODO: We could save time later by mangling the
# structure to remove seeds we don't care about.
else:
logging.warning(
"Skipping empty seed structure for %s.%s",
flavour, series_name)
except SeedError as e:
logging.warning(
"Failed to fetch seeds for %s.%s: %s",
flavour, series_name, e)
return structures
def germinateArchFlavour(self, germinator, suite, arch, flavour, structure,
seed_names):
"""Germinate seeds on a single flavour for a single architecture."""
germinator.plant_seeds(structure)
germinator.grow(structure)
germinator.add_extras(structure)
# Unfortunately we have to use several bits of Germinate internals
# here. I promise not to change them under myself without notifying
# myself.
all_seeds = OrderedDict()
for seed_name in seed_names:
seed = germinator._get_seed(structure, seed_name)
for inner_seed in germinator._inner_seeds(seed):
if inner_seed.name not in all_seeds:
all_seeds[inner_seed.name] = inner_seed
if "extra-packages" in structure:
seed = germinator._get_seed(structure, "extra-packages")
for inner_seed in germinator._inner_seeds(seed):
if inner_seed.name not in all_seeds:
all_seeds[inner_seed.name] = inner_seed
for seed in list(all_seeds.values()):
sources = seed._sourcepkgs | seed._build_sourcepkgs
for source in sources:
version = germinator._sources[source]["Version"]
if (source in self._versions and
self._versions[source] != version):
# This requires manual investigation, as the resulting
# derived distribution series can only have one version
# of any given source package.
raise Exception(
"Conflicting source versions: seed %s/%s requires "
"%s %s, but already present at version %s" % (
flavour, seed, source, version,
self._versions[source]))
self._versions[source] = version
def checkImageManifest(self, germinator, arch, manifest_url):
ok = True
with open_url_as_text(manifest_url) as manifest:
for line in manifest:
try:
package, version = line.split()
if package.startswith("click:"):
continue
package = package.split(":", 1)[0]
if package not in germinator._packages:
raise ManifestError(
"%s not found for %s (from %s)" % (
package, arch, manifest_url))
gpkg = germinator._packages[package]
if gpkg["Version"] != version:
raise ManifestError(
"Found %s %s for %s, but wanted %s "
"(from %s)" % (
package, gpkg["Version"], arch, version,
manifest_url))
if gpkg["Source"] not in self._versions:
raise ManifestError(
"%s not copied (from %s)" % (
gpkg["Source"], manifest_url))
except ManifestError as e:
logging.error(e.message)
ok = False
return ok
def germinateArch(self, suite, components, arch, flavours, structures):
"""Germinate seeds on all flavours for a single architecture."""
germinator = Germinator(arch)
# Read archive metadata.
logging.info("Reading archive for %s/%s", suite, arch)
archive = TagFile(suite, components, arch, self.mirror, cleanup=True)
germinator.parse_archive(archive)
if self.options.all_packages:
for source in germinator._sources:
self._versions[source] = germinator._sources[source]["Version"]
else:
for flavour, seed_names in list(flavours.items()):
logging.info("Germinating for %s/%s/%s", flavour, suite, arch)
self.germinateArchFlavour(
germinator, suite, arch, flavour, structures[flavour],
seed_names)
ok = True
if self.options.check_image_manifest:
for manifest_id in self.options.check_image_manifest:
manifest_arch, manifest_url = manifest_id.split(":", 1)
if arch != manifest_arch:
continue
if not self.checkImageManifest(germinator, arch, manifest_url):
ok = False
return ok
def getVersions(self, full_seed_names, extra_packages):
self._versions = {}
suite = self.options.suite
components = self.components
architectures = [
a.architecture_tag for a in self.options.architectures]
flavours = OrderedDict()
for full_seed_name in full_seed_names:
flavour, seed_name = full_seed_name.split("/")
flavours.setdefault(flavour, []).append(seed_name)
if self.options.all_packages:
structures = None
else:
logging.info("Reading seed structures")
structures = self.makeSeedStructures(
suite, flavours, extra_packages)
if self.options.all_packages or structures:
ok = True
for arch in architectures:
if not self.germinateArch(
suite, components, arch, flavours, structures):
ok = False
if not ok:
sys.exit(1)
return self._versions
def retry_on_error(func, *args, **kwargs):
# Since failure will be expensive to restart from scratch, we try this a
# few times in case of failure.
for i in range(3):
try:
return func(*args, **kwargs)
except HTTPError as e:
print(e.content, file=sys.stderr)
if i == 2:
raise
time.sleep(15)
def derive_distribution(options, args):
full_seed_names = [
arg[len("seed:"):] for arg in args if arg.startswith("seed:")]
if not full_seed_names and not options.all_packages:
raise Exception(
"You must specify at least one seed name (in the form "
"seed:COLLECTION/NAME).")
extra_packages = [arg for arg in args if not arg.startswith("seed:")]
ttg = TimeTravellingGerminator(options)
versions = ttg.getVersions(full_seed_names, extra_packages)
if options.excludes:
for exclude in options.excludes:
versions.pop(exclude, None)
# Skip anything we already have, to simplify incremental copies.
original_versions = dict(versions)
removable = {}
newer = {}
for spph in options.destination.archive.getPublishedSources(
distro_series=options.destination.series,
pocket=options.destination.pocket, status="Published"):
source = spph.source_package_name
if source not in versions:
removable[source] = spph.source_package_version
else:
diff = apt_pkg.version_compare(
versions[source], spph.source_package_version)
if diff < 0:
newer[source] = spph
elif diff == 0:
del versions[source]
print("Copy candidates:")
for source, version in sorted(versions.items()):
print("\t%s\t%s" % (source, version))
print()
if newer:
print("These packages previously had newer version numbers:")
for source, spph in sorted(newer.items()):
print(
"\t%s\t%s -> %s" % (
source, spph.source_package_version, versions[source]))
print()
if removable:
print("These packages could possibly be removed:")
for source, version in sorted(removable.items()):
print("\t%s\t%s" % (source, version))
print()
if options.dry_run:
print("Dry run; no packages copied.")
else:
if YesNoQuestion().ask("Copy", "no") == "no":
return False
print("Setting packaging information ...")
for source in sorted(versions.keys()):
sp = options.series.getSourcePackage(name=source)
if sp.productseries_link is not None:
derived_sp = options.destination.series.getSourcePackage(
name=source)
if derived_sp.productseries_link is None:
retry_on_error(
derived_sp.setPackaging,
productseries=sp.productseries_link)
print(".", end="")
sys.stdout.flush()
print()
# Wouldn't it be lovely if we could do a single copyPackages call
# with a giant dictionary of source package names to versions? As
# it is we need to call copyPackage a few thousand times instead.
archive = options.destination.archive
for source, version in sorted(versions.items()):
print("\t%s\t%s" % (source, version))
if source in newer:
retry_on_error(
newer[source].requestDeletion,
removal_comment=(
"derive-distribution rewinding to %s" % version))
retry_on_error(
archive.copyPackage,
source_name=source, version=version,
from_archive=options.archive,
to_pocket=options.destination.pocket,
to_series=options.destination.series.name,
include_binaries=True, auto_approve=True, silent=True)
print("Checking package sets ...")
found_any_set = False
for source_set in options.launchpad.packagesets.getBySeries(
distroseries=options.series):
sources = source_set.getSourcesIncluded(direct_inclusion=True)
if set(sources) & set(original_versions):
print("\t%s" % source_set.name)
found_any_set = True
if found_any_set:
print(
"A member of ~techboard needs to copy the above package sets "
"to the new series.")
return True
def main():
logging.basicConfig(level=logging.INFO)
logging.getLogger("germinate").setLevel(logging.CRITICAL)
apt_pkg.init()
parser = OptionParser(
usage="usage: %prog --to-distribution distribution [options]")
parser.add_option(
"-l", "--launchpad", dest="launchpad_instance", default="production")
parser.add_option(
"-n", "--dry-run", default=False, action="store_true",
help="only show actions that would be performed")
parser.add_option(
"-d", "--distribution", default="ubuntu",
metavar="DISTRIBUTION", help="copy from DISTRIBUTION")
parser.add_option("-s", "--suite", metavar="SUITE", help="copy from SUITE")
parser.add_option(
"-a", "--architecture", dest="architectures", action="append",
metavar="ARCHITECTURE",
help="architecture tag (may be given multiple times)")
parser.add_option(
"--to-distribution", metavar="DISTRIBUTION",
help="copy to DISTRIBUTION")
parser.add_option(
"--to-suite", metavar="SUITE",
help="copy to SUITE (default: copy from suite)")
parser.add_option(
"--date", metavar="DATE", help=(
"copy from suite as it existed on DATE; assumes UTC if timezone "
"not specified"))
parser.add_option("--seed-source", help="fetch seeds from SOURCE")
parser.add_option(
"--check-image-manifest", action="append", metavar="ARCH:URL", help=(
"ensure that all packages from the manifest at URL for "
"architecture ARCH are copied (may be given multiple times)"))
parser.add_option(
"--exclude", dest="excludes", action="append", metavar="PACKAGE",
help="don't copy PACKAGE (may be given multiple times)")
parser.add_option(
"--all-packages", default=False, action="store_true",
help="copy all packages in source suite rather than germinating")
options, args = parser.parse_args()
if not args and not options.all_packages:
parser.error("You must specify some seeds or packages to copy.")
if options.launchpad_instance == "dogfood":
# Work around old service root in some versions of launchpadlib.
options.launchpad_instance = "https://api.dogfood.paddev.net/"
options.launchpad = Launchpad.login_with(
"derive-distribution", options.launchpad_instance, version="devel")
lputils.setup_location(options)
options.destination = Values()
options.destination.launchpad = options.launchpad
options.destination.distribution = options.to_distribution
options.destination.suite = options.to_suite
options.destination.architectures = [
a.architecture_tag for a in options.architectures]
# In cases where source is specified, but destination is not, default to
# destination = source.
if options.destination.distribution is None:
options.destination.distribution = options.distribution
if options.destination.suite is None:
options.destination.suite = options.suite
if options.date is not None:
options.date = dateutil_parser.parse(options.date)
if options.date.tzinfo is None:
options.date = options.date.replace(tzinfo=pytz.UTC)
lputils.setup_location(options.destination)
if (options.distribution == options.destination.distribution and
options.suite == options.destination.suite):
parser.error("copy destination must differ from source")
if derive_distribution(options, args):
return 0
else:
return 1
if __name__ == "__main__":
sys.exit(main())
|