← Back to branch summary

~bjornt/launchpad/architect-vision

« back to all changes in this revision

Viewing changes to lib/lp/buildmaster/master.py

  • Committer: Bjorn Tillenius
  • Date: 2010-05-12 12:52:09 UTC
  • mfrom: (10599.1.251 launchpad)
  • Revision ID: bjorn@canonical.com-20100512125209-pip7jav1a7xq52d8
Merge RF.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/buildmaster/master.py
1
 
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2
 
# GNU Affero General Public License version 3 (see the file LICENSE).
3
 
 
4
 
#Authors: Daniel Silverstone <daniel.silverstone@canonical.com>
5
 
#         Celso Providelo <celso.providelo@canonical.com>
6
 
 
7
 
"""Common code for Buildd scripts
8
 
 
9
 
Module used by buildd-queue-builder.py and buildd-slave-scanner.py
10
 
cronscripts.
11
 
"""
12
 
 
13
 
__metaclass__ = type
14
 
 
15
 
 
16
 
import logging
17
 
 
18
 
from zope.component import getUtility
19
 
 
20
 
from canonical.buildd.utils import notes
21
 
from canonical.config import config
22
 
from canonical.librarian.interfaces import ILibrarianClient
23
 
from lp.archivepublisher.utils import process_in_batches
24
 
from lp.buildmaster.interfaces.buildbase import BuildStatus
25
 
from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
26
 
from lp.buildmaster.buildergroup import BuilderGroup
27
 
from lp.soyuz.interfaces.build import IBuildSet
28
 
from lp.soyuz.pas import BuildDaemonPackagesArchSpecific
29
 
 
30
 
 
31
 
class BuilddMaster:
32
 
    """Canonical autobuilder master, toolkit and algorithms.
33
 
 
34
 
    This class is in the process of being deprecated in favour of the regular
35
 
    content classes.
36
 
    """
37
 
    # XXX cprov 2007-06-15: Please do not extend this class except as
38
 
    # required to move more logic into the content classes. A new feature
39
 
    # should be modeled directly in IBuilder.
40
 
 
41
 
    def __init__(self, logger, tm):
42
 
        self._logger = logger
43
 
        self._tm = tm
44
 
        self.librarian = getUtility(ILibrarianClient)
45
 
        self._archseries = {}
46
 
        self._logger.debug("Buildd Master has been initialised")
47
 
 
48
 
    def commit(self):
49
 
        self._tm.commit()
50
 
 
51
 
    def addDistroArchSeries(self, distroarchseries):
52
 
        """Setting up a workable DistroArchSeries for this session."""
53
 
        self._logger.debug("Adding DistroArchSeries %s/%s/%s"
54
 
                          % (distroarchseries.distroseries.distribution.name,
55
 
                             distroarchseries.distroseries.name,
56
 
                             distroarchseries.architecturetag))
57
 
 
58
 
        # Is there a chroot for this archseries?
59
 
        if distroarchseries.getChroot():
60
 
            # Fill out the contents.
61
 
            self._archseries.setdefault(distroarchseries, {})
62
 
 
63
 
    def setupBuilders(self, archseries):
64
 
        """Setting up a group of builder slaves for a given DistroArchSeries.
65
 
 
66
 
        Use the annotation utility to store a BuilderGroup instance
67
 
        keyed by the the DistroArchSeries.processorfamily in the
68
 
        global registry 'notes' and refer to this 'note' in the private
69
 
        attribute '_archseries' keyed by the given DistroArchSeries
70
 
        and the label 'builders'. This complicated arrangement enables us
71
 
        to share builder slaves between different DistroArchRelases since
72
 
        their processorfamily values are the same (compatible processors).
73
 
        """
74
 
        # Determine the builders for this distroarchseries...
75
 
        if archseries not in self._archseries:
76
 
            # Avoid entering in the huge loop if we don't find at least
77
 
            # one architecture for which we can build on.
78
 
            self._logger.debug(
79
 
                "Chroot missing for %s/%s/%s, skipping"
80
 
                % (archseries.distroseries.distribution.name,
81
 
                   archseries.distroseries.name,
82
 
                   archseries.architecturetag))
83
 
            return
84
 
 
85
 
        # query the global annotation registry and verify if
86
 
        # we have already done the builder checks for the
87
 
        # processor family in question. if it's already done
88
 
        # simply refer to that information in the _archseries
89
 
        # attribute.
90
 
        if 'builders' not in notes[archseries.processorfamily]:
91
 
 
92
 
            # setup a BuilderGroup object
93
 
            info = "builders.%s" % archseries.processorfamily.name
94
 
            builderGroup = BuilderGroup(self.getLogger(info), self._tm)
95
 
 
96
 
            # check the available slaves for this archseries
97
 
            builderGroup.checkAvailableSlaves(archseries)
98
 
 
99
 
            # annotate the group of builders for the
100
 
            # DistroArchSeries.processorfamily in question and the
101
 
            # label 'builders'
102
 
            notes[archseries.processorfamily]["builders"] = builderGroup
103
 
 
104
 
        # consolidate the annotation for the architecture release
105
 
        # in the private attribute _archreleases
106
 
        self._archseries[archseries]["builders"] = \
107
 
            notes[archseries.processorfamily]["builders"]
108
 
 
109
 
    def createMissingBuilds(self, distroseries):
110
 
        """Ensure that each published package is completly built."""
111
 
        self._logger.info("Processing %s" % distroseries.name)
112
 
        # Do not create builds for distroseries with no nominatedarchindep
113
 
        # they can't build architecture independent packages properly.
114
 
        if not distroseries.nominatedarchindep:
115
 
            self._logger.debug(
116
 
                "No nominatedarchindep for %s, skipping" % distroseries.name)
117
 
            return
118
 
 
119
 
        # Listify the architectures to avoid hitting this MultipleJoin
120
 
        # multiple times.
121
 
        distroseries_architectures = list(distroseries.architectures)
122
 
        if not distroseries_architectures:
123
 
            self._logger.debug(
124
 
                "No architectures defined for %s, skipping"
125
 
                % distroseries.name)
126
 
            return
127
 
 
128
 
        architectures_available = list(distroseries.enabled_architectures)
129
 
        if not architectures_available:
130
 
            self._logger.debug(
131
 
                "Chroots missing for %s, skipping" % distroseries.name)
132
 
            return
133
 
 
134
 
        self._logger.info(
135
 
            "Supported architectures: %s" %
136
 
            " ".join(arch_series.architecturetag
137
 
                     for arch_series in architectures_available))
138
 
 
139
 
        pas_verify = BuildDaemonPackagesArchSpecific(
140
 
            config.builddmaster.root, distroseries)
141
 
 
142
 
        sources_published = distroseries.getSourcesPublishedForAllArchives()
143
 
        self._logger.info(
144
 
            "Found %d source(s) published." % sources_published.count())
145
 
 
146
 
        def process_source(pubrec):
147
 
            builds = pubrec.createMissingBuilds(
148
 
                architectures_available=architectures_available,
149
 
                pas_verify=pas_verify, logger=self._logger)
150
 
            if len(builds) > 0:
151
 
                self.commit()
152
 
 
153
 
        process_in_batches(
154
 
            sources_published, process_source, self._logger,
155
 
            minimum_chunk_size=1000)
156
 
 
157
 
    def addMissingBuildQueueEntries(self):
158
 
        """Create missing Buildd Jobs. """
159
 
        self._logger.info("Scanning for build queue entries that are missing")
160
 
 
161
 
        buildset = getUtility(IBuildSet)
162
 
        builds = buildset.getPendingBuildsForArchSet(self._archseries)
163
 
 
164
 
        if not builds:
165
 
            return
166
 
 
167
 
        for build in builds:
168
 
            if not build.buildqueue_record:
169
 
                name = build.sourcepackagerelease.name
170
 
                version = build.sourcepackagerelease.version
171
 
                tag = build.distroarchseries.architecturetag
172
 
                self._logger.debug(
173
 
                    "Creating buildqueue record for %s (%s) on %s"
174
 
                    % (name, version, tag))
175
 
                build.queueBuild()
176
 
 
177
 
        self.commit()
178
 
 
179
 
    def scanActiveBuilders(self):
180
 
        """Collect informations/results of current build jobs."""
181
 
 
182
 
        queueItems = getUtility(IBuildQueueSet).getActiveBuildJobs()
183
 
 
184
 
        self._logger.debug(
185
 
            "scanActiveBuilders() found %d active build(s) to check"
186
 
            % queueItems.count())
187
 
 
188
 
        for job in queueItems:
189
 
            job.builder.updateBuild(job)
190
 
            self.commit()
191
 
 
192
 
    def getLogger(self, subname=None):
193
 
        """Return the logger instance with specific prefix"""
194
 
        if subname is None:
195
 
            return self._logger
196
 
        return logging.getLogger("%s.%s" % (self._logger.name, subname))
197
 
 
198
 
    def scoreCandidates(self):
199
 
        """Iterate over the pending buildqueue entries and re-score them."""
200
 
        if not self._archseries:
201
 
            self._logger.info("No architecture found to rescore.")
202
 
            return
203
 
 
204
 
        # Get the current build job candidates.
205
 
        archseries = self._archseries.keys()
206
 
        bqset = getUtility(IBuildQueueSet)
207
 
        candidates = bqset.calculateCandidates(archseries)
208
 
 
209
 
        self._logger.info("Found %d build in NEEDSBUILD state. Rescoring"
210
 
                          % candidates.count())
211
 
 
212
 
        for job in candidates:
213
 
            uptodate_build = getUtility(IBuildSet).getByQueueEntry(job)
214
 
            if uptodate_build.buildstate != BuildStatus.NEEDSBUILD:
215
 
                continue
216
 
            job.score()
217
 
 
218
 
        self.commit()