1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
|
#! /usr/bin/python
# Based on lib/lp/archivepublisher/scripts/generate_extra_overrides.py from
# Launchpad, which is:
#
# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
import copy
from functools import partial
import glob
import logging
from optparse import OptionParser
import os
import pickle
import shutil
import sys
import traceback
from germinate.archive import TagFile
from germinate.germinator import Germinator
from germinate.log import GerminateFormatter
from germinate.seeds import SeedError, SeedStructure
from launchpadlib.launchpad import Launchpad
import lputils
class AtomicFile:
"""Facilitate atomic writing of files."""
def __init__(self, filename):
self.filename = filename
self.fd = open("%s.new" % self.filename, "w")
def __enter__(self):
return self.fd
def __exit__(self, exc_type, exc_value, exc_tb):
self.fd.close()
if exc_type is None:
os.rename("%s.new" % self.filename, self.filename)
class BufferHandler(logging.Handler):
"""A log handler which stores records for emission by another logger."""
def __init__(self):
super(BufferHandler, self).__init__()
self.records = []
def emit(self, record):
# Record arguments may not be available at the other end.
record_copy = copy.copy(record)
record_copy.msg = record.getMessage()
record_copy.args = None
self.records.append(pickle.dumps(record_copy, -1))
class ExtraGerminate:
def __init__(self, args):
self.germinate_logger = None
self.raw_args = args
def processOptions(self):
"""Handle command-line options."""
parser = OptionParser()
parser.add_option(
"-l", "--launchpad", dest="launchpad_instance",
default="production")
parser.add_option(
"-d", "--distribution", dest="distribution", metavar="DISTRO",
default="ubuntu",
help="look in distribution DISTRO")
parser.add_option("-s", "--suite", help="process this suite")
parser.add_option(
"-o", "--output", default="germinate-for-proposed",
help="output to this directory")
parser.add_option(
"--archiveroot",
default="file://%s" % os.path.expanduser("~/mirror/ubuntu/"),
help="distribution archive URL")
self.options, self.args = parser.parse_args(self.raw_args)
self.options.launchpad = Launchpad.login_anonymously(
'germinate-for-proposed', self.options.launchpad_instance)
lputils.setup_location(self.options, default_pocket="Proposed")
self.germinateroot = "%s.new" % self.options.output
def setUpDirs(self):
"""Create output directories if they did not already exist."""
if os.path.exists(self.germinateroot):
shutil.rmtree(self.germinateroot)
logging.debug("Creating germinate root %s." % self.germinateroot)
os.makedirs(self.germinateroot)
def addLogHandler(self):
"""Send germinate's log output to a separate file."""
if self.germinate_logger is not None:
return
self.germinate_logger = logging.getLogger("germinate")
self.germinate_logger.setLevel(logging.INFO)
self.log_file = os.path.join(self.germinateroot, "germinate.output")
self.log_handler = logging.FileHandler(self.log_file, mode="w")
self.log_handler.setFormatter(GerminateFormatter())
self.germinate_logger.addHandler(self.log_handler)
self.germinate_logger.propagate = False
def setUp(self):
"""Process options, and set up internal state."""
self.processOptions()
self.setUpDirs()
self.addLogHandler()
def getComponents(self, series):
"""Get the list of components to process for a given distroseries."""
return ["main", "restricted", "universe", "multiverse"]
def makeSeedStructures(self, suite, flavours, seed_bases=None):
series_name = suite.split("-")[0]
structures = {}
for flavour in flavours:
try:
structure = SeedStructure(
"%s.%s" % (flavour, series_name), seed_bases=seed_bases)
if len(structure):
structures[flavour] = structure
else:
logging.warning(
"Skipping empty seed structure for %s.%s",
flavour, series_name)
except SeedError as e:
logging.warning(
"Failed to fetch seeds for %s.%s: %s",
flavour, series_name, e)
return structures
def logGerminateProgress(self, *args):
"""Log a "progress" entry to the germinate log file.
Germinate logs quite a bit of detailed information. To make it
easier to see the structure of its operation, GerminateFormatter
allows tagging some log entries as "progress" entries, which are
printed without a prefix.
"""
self.germinate_logger.info(*args, extra={"progress": True})
def composeOutputPath(self, flavour, series_name, arch, base):
return os.path.join(
self.germinateroot,
"%s_%s_%s_%s" % (base, flavour, series_name, arch))
def recordOutput(self, path, seed_outputs):
if seed_outputs is not None:
seed_outputs.add(os.path.basename(path))
def writeGerminateOutput(self, germinator, structure, flavour,
suite, arch, seed_outputs=None):
"""Write dependency-expanded output files.
These files are a reduced subset of those written by the germinate
command-line program.
"""
series_name = suite.split("-")[0]
path = partial(self.composeOutputPath, flavour, series_name, arch)
# The structure file makes it possible to figure out how the other
# output files relate to each other.
structure.write(path("structure"))
self.recordOutput(path("structure"), seed_outputs)
# "all" and "all.sources" list the full set of binary and source
# packages respectively for a given flavour/suite/architecture
# combination.
germinator.write_all_list(structure, path("all"))
self.recordOutput(path("all"), seed_outputs)
germinator.write_all_source_list(structure, path("all.sources"))
self.recordOutput(path("all.sources"), seed_outputs)
# Write the dependency-expanded output for each seed. Several of
# these are used by archive administration tools, and others are
# useful for debugging, so it's best to just write them all.
for seedname in structure.names:
germinator.write_full_list(structure, path(seedname), seedname)
self.recordOutput(path(seedname), seed_outputs)
def germinateArchFlavour(self, germinator, suite, arch, flavour,
structure, primary_flavour, seed_outputs=None):
"""Germinate seeds on a single flavour for a single architecture."""
# Expand dependencies.
germinator.plant_seeds(structure)
germinator.grow(structure)
germinator.add_extras(structure)
self.writeGerminateOutput(
germinator, structure, flavour, suite, arch,
seed_outputs=seed_outputs)
def germinateArch(self, suite, components, arch, flavours, structures):
"""Germinate seeds on all flavours for a single architecture."""
# Buffer log output for each architecture so that it appears
# sequential.
self.germinate_logger.removeHandler(self.log_handler)
log_handler = BufferHandler()
self.germinate_logger.addHandler(log_handler)
germinator = Germinator(arch)
# Compute ogre model.
if "-" in suite:
series, pocket = suite.split("-")
suites = [series]
if pocket in ("updates", "security", "proposed"):
suites.append("%s-security" % series)
if pocket in ("updates", "proposed"):
suites.append("%s-updates" % series)
if pocket == "proposed":
suites.append("%s-proposed" % series)
else:
suites = [suite]
# Read archive metadata.
archive = TagFile(
suites, components, arch, self.options.archiveroot, cleanup=True)
germinator.parse_archive(archive)
seed_outputs = set()
for flavour in flavours:
logging.info(
"Germinating for %s/%s/%s", flavour, suite, arch)
# Add this to the germinate log as well so that that can be
# debugged more easily. Log a separator line first.
self.logGerminateProgress("")
self.logGerminateProgress(
"Germinating for %s/%s/%s", flavour, suite, arch)
self.germinateArchFlavour(
germinator, suite, arch, flavour,
structures[flavour], flavour == flavours[0],
seed_outputs=seed_outputs)
return log_handler.records, seed_outputs
def germinateArchChild(self, close_in_child, wfd, *args):
"""Helper method to call germinateArch in a forked child process."""
try:
for fd in close_in_child:
os.close(fd)
with os.fdopen(wfd, "wb") as writer:
pickle.dump(self.germinateArch(*args), writer, -1)
return 0
except:
traceback.print_exc()
pickle.dump(([], set()), writer, -1)
return 1
def removeStaleOutputs(self, suite, seed_outputs):
"""Remove stale outputs for a suite.
Any per-seed outputs not in seed_outputs are considered stale.
"""
series_name = suite.split("-")[0]
all_outputs = glob.glob(
os.path.join(self.germinateroot, "*_*_%s_*" % series_name))
for output in all_outputs:
if os.path.basename(output) not in seed_outputs:
os.remove(output)
def generateExtraOverrides(self, suite, components, architectures,
flavours, seed_bases=None):
structures = self.makeSeedStructures(
suite, flavours, seed_bases=seed_bases)
if structures:
procs = []
close_in_child = []
for arch in architectures:
rfd, wfd = os.pipe()
close_in_child.append(rfd)
pid = os.fork()
if pid == 0: # child
os._exit(self.germinateArchChild(
close_in_child, wfd,
suite, components, arch, flavours, structures))
else: # parent
os.close(wfd)
reader = os.fdopen(rfd, "rb")
procs.append((pid, reader))
seed_outputs = set()
for pid, reader in procs:
log_records, arch_seed_outputs = pickle.load(reader)
for log_record in log_records:
self.germinate_logger.handle(pickle.loads(log_record))
seed_outputs |= arch_seed_outputs
reader.close()
os.waitpid(pid, 0)
self.removeStaleOutputs(suite, seed_outputs)
def process(self, seed_bases=None):
"""Do the bulk of the work."""
self.setUp()
components = self.getComponents(self.options.series)
architectures = [
a.architecture_tag for a in self.options.architectures]
self.generateExtraOverrides(
self.options.suite, components, architectures, self.args,
seed_bases=seed_bases)
if os.path.exists("%s.old" % self.options.output):
shutil.rmtree("%s.old" % self.options.output)
if os.path.exists(self.options.output):
os.rename(self.options.output, "%s.old" % self.options.output)
os.rename(self.germinateroot, self.options.output)
if os.path.exists("%s.old" % self.options.output):
shutil.rmtree("%s.old" % self.options.output)
if __name__ == "__main__":
ExtraGerminate(sys.argv[1:]).process()
|