← Back to branch summary

~ubuntu-archive/ubuntu-archive-scripts/trunk

« back to all changes in this revision

Viewing changes to extra-germinate

  • Committer: Colin Watson
  • Date: 2013-08-07 11:43:00 UTC
  • Revision ID: cjwatson@canonical.com-20130807114300-9b2cu8o2wlc12upj
run-phased-updater: make bzr quieter

Show diffs side-by-side

added added

removed removed

Lines of Context:
extra-germinate
1
 
#! /usr/bin/python3
 
1
#! /usr/bin/python
2
2
 
3
3
# Based on lib/lp/archivepublisher/scripts/generate_extra_overrides.py from
4
4
# Launchpad, which is:
6
6
# Copyright 2011-2012 Canonical Ltd.  This software is licensed under the
7
7
# GNU Affero General Public License version 3 (see the file LICENSE).
8
8
 
9
 
import copy
10
9
from functools import partial
11
10
import glob
12
11
import logging
13
12
from optparse import OptionParser
14
13
import os
15
 
import pickle
16
14
import shutil
17
15
import sys
18
 
import traceback
19
16
 
20
17
from germinate.archive import TagFile
21
18
from germinate.germinator import Germinator
41
38
            os.rename("%s.new" % self.filename, self.filename)
42
39
 
43
40
 
44
 
class BufferHandler(logging.Handler):
45
 
    """A log handler which stores records for emission by another logger."""
46
 
 
47
 
    def __init__(self):
48
 
        super(BufferHandler, self).__init__()
49
 
        self.records = []
50
 
 
51
 
    def emit(self, record):
52
 
        # Record arguments may not be available at the other end.
53
 
        record_copy = copy.copy(record)
54
 
        record_copy.msg = record.getMessage()
55
 
        record_copy.args = None
56
 
        self.records.append(pickle.dumps(record_copy, -1))
57
 
 
58
 
 
59
41
class ExtraGerminate:
60
42
    def __init__(self, args):
61
43
        self.germinate_logger = None
101
83
        self.germinate_logger = logging.getLogger("germinate")
102
84
        self.germinate_logger.setLevel(logging.INFO)
103
85
        self.log_file = os.path.join(self.germinateroot, "germinate.output")
104
 
        self.log_handler = logging.FileHandler(self.log_file, mode="w")
105
 
        self.log_handler.setFormatter(GerminateFormatter())
106
 
        self.germinate_logger.addHandler(self.log_handler)
 
86
        handler = logging.FileHandler(self.log_file, mode="w")
 
87
        handler.setFormatter(GerminateFormatter())
 
88
        self.germinate_logger.addHandler(handler)
107
89
        self.germinate_logger.propagate = False
108
90
 
109
91
    def setUp(self):
196
178
            germinator, structure, flavour, suite, arch,
197
179
            seed_outputs=seed_outputs)
198
180
 
199
 
    def germinateArch(self, suite, components, arch, flavours, structures):
 
181
    def germinateArch(self, suite, components, arch, flavours,
 
182
                      structures, seed_outputs=None):
200
183
        """Germinate seeds on all flavours for a single architecture."""
201
 
        # Buffer log output for each architecture so that it appears
202
 
        # sequential.
203
 
        self.germinate_logger.removeHandler(self.log_handler)
204
 
        log_handler = BufferHandler()
205
 
        self.germinate_logger.addHandler(log_handler)
206
 
 
207
184
        germinator = Germinator(arch)
208
185
 
209
186
        # Compute ogre model.
224
201
            suites, components, arch, self.options.archiveroot, cleanup=True)
225
202
        germinator.parse_archive(archive)
226
203
 
227
 
        seed_outputs = set()
228
204
        for flavour in flavours:
229
205
            logging.info(
230
206
                "Germinating for %s/%s/%s", flavour, suite, arch)
239
215
                structures[flavour], flavour == flavours[0],
240
216
                seed_outputs=seed_outputs)
241
217
 
242
 
        return log_handler.records, seed_outputs
243
 
 
244
 
    def germinateArchChild(self, close_in_child, wfd, *args):
245
 
        """Helper method to call germinateArch in a forked child process."""
246
 
        try:
247
 
            for fd in close_in_child:
248
 
                os.close(fd)
249
 
            with os.fdopen(wfd, "wb") as writer:
250
 
                pickle.dump(self.germinateArch(*args), writer, -1)
251
 
            return 0
252
 
        except:
253
 
            traceback.print_exc()
254
 
            pickle.dump(([], set()), writer, -1)
255
 
            return 1
256
 
 
257
218
    def removeStaleOutputs(self, suite, seed_outputs):
258
219
        """Remove stale outputs for a suite.
259
220
 
272
233
            suite, flavours, seed_bases=seed_bases)
273
234
 
274
235
        if structures:
275
 
            procs = []
276
 
            close_in_child = []
 
236
            seed_outputs = set()
277
237
            for arch in architectures:
278
 
                rfd, wfd = os.pipe()
279
 
                close_in_child.append(rfd)
280
 
                pid = os.fork()
281
 
                if pid == 0:  # child
282
 
                    os._exit(self.germinateArchChild(
283
 
                        close_in_child, wfd,
284
 
                        suite, components, arch, flavours, structures))
285
 
                else:  # parent
286
 
                    os.close(wfd)
287
 
                    reader = os.fdopen(rfd, "rb")
288
 
                    procs.append((pid, reader))
289
 
 
290
 
            seed_outputs = set()
291
 
            for pid, reader in procs:
292
 
                log_records, arch_seed_outputs = pickle.load(reader)
293
 
                for log_record in log_records:
294
 
                    self.germinate_logger.handle(pickle.loads(log_record))
295
 
                seed_outputs |= arch_seed_outputs
296
 
                reader.close()
297
 
                os.waitpid(pid, 0)
 
238
                self.germinateArch(
 
239
                    suite, components, arch, flavours,
 
240
                    structures, seed_outputs=seed_outputs)
298
241
            self.removeStaleOutputs(suite, seed_outputs)
299
242
 
300
243
    def process(self, seed_bases=None):