~xnox/ubuntu-archive-tools/sru-report-autopkgtest-vomit

#! /usr/bin/python3

# Copyright (C) 2011, 2012  Canonical Ltd.

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; version 3 of the License.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# This script can be used to reschedule some of the copy archives

# builds so that they are processed like regular PPA builds.

#

# Copy archives builds have a huge penalty applied to them which means

# that they are only processed when there is nothing else being processed

# by the build farm. That's usually fine, but for some rebuilds, we want

# more timely processing, while at the same time, we do want to continue to

# service regular PPA builds.

#

# This script will try to have a portion of the build farm processing copy

# builds. It does that by rescoring builds to the normal build priority

# range. But will only rescore a few builds at a time, so as not to take ove

# the build pool. By default, it won't rescore more than 1/4 the number of

# available builders. So for example, if there are 12 i386 builders, only

# 3 builds at a time will have a "normal priority".

import argparse

from collections import defaultdict

import logging

import time

from launchpadlib.launchpad import Launchpad

API_NAME = 'copy-build-scheduler'

NEEDS_BUILDING = 'Needs building'

BUILDING = 'Currently building'

COPY_ARCHIVE_SCORE_PENALTY = 2600

# Number of minutes to wait between schedule run.

SCHEDULE_PERIOD = 5

def determine_builder_capacity(lp, args):

    """Find how many builders to use for copy builds by processor."""

    capacity = {}

    for processor in args.processors:

        queue = [

            builder for builder in lp.builders.getBuildersForQueue(

                processor='/+processors/%s' % processor, virtualized=True)

            if builder.active]

        max_capacity = len(queue)

        capacity[processor] = round(max_capacity * args.builder_ratio)

        # Make sure at least 1 builders is associated

        if capacity[processor] == 0:

            capacity[processor] = 1

        logging.info(

            'Will use %d out of %d %s builders', capacity[processor],

            max_capacity, processor)

    return capacity

def get_archive_used_builders_capacity(archive):

    """Return the number of builds currently being done for the archive."""

    capacity = defaultdict(int)

    building = archive.getBuildRecords(build_state=BUILDING)

    for build in building:

        capacity[build.arch_tag] += 1

    return capacity

def main():

    parser = argparse.ArgumentParser()

    parser.add_argument(

        '--lp-instance', default='production', dest='lp_instance',

        help="Select the Launchpad instance to run against. Defaults to "

        "'production'")

    parser.add_argument(

        '-v', '--verbose', default=0, action='count', dest='verbose',

        help="Increase verbosity of the script. -v prints info messages"

        "-vv will print debug messages.")

    parser.add_argument(

        '-c', '--credentials', default=None, action='store',

        dest='credentials',

        help="Use the OAuth credentials in FILE instead of the desktop "

        "one.", metavar='FILE')

    parser.add_argument(

        '-d', '--distribution', default='ubuntu', action='store',

        dest='distribution',

        help="The archive distribution. Defaults to 'ubuntu'.")

    parser.add_argument(

        '-p', '--processor', action='append', dest='processors',

        help="The processor for which to schedule builds. "

        "Default to i386 and amd64.")

    parser.add_argument(

        '-r', '--ratio', default=0.25, action='store', type=float,

        dest='builder_ratio',

        help="The ratio of builders that you want to use for the copy "

        "builds.  Default to 25%% of the available builders.")

    parser.add_argument('copy_archive_name', help='Name of copy archive')

    args = parser.parse_args()

    if args.verbose >= 2:

        log_level = logging.DEBUG

    elif args.verbose == 1:

        log_level = logging.INFO

    else:

        log_level = logging.WARNING

    logging.basicConfig(level=log_level)

    if args.builder_ratio >= 1 or args.builder_ratio < 0:

        parser.error(

            'ratio should be a float between 0 and 1: %s' %

            args.builder_ratio)

    if not args.processors:

        args.processors = ['amd64', 'i386']

    lp = Launchpad.login_with(

        API_NAME, args.lp_instance,

        credentials_file=args.credentials,

        version='devel')

    try:

        distribution = lp.distributions[args.distribution]

    except KeyError:

        parser.error('unknown distribution: %s' % args.distribution)

    archive = distribution.getArchive(name=args.copy_archive_name)

    if archive is None:

        parser.error('unknown archive: %s' % args.copy_archive_name)

    iteration = 0

    while True:

        # Every 5 schedules run - and on the first - compute available

        # capacity.

        if (iteration % 5) == 0:

            capacity = determine_builder_capacity(lp, args)

        iteration += 1

        pending_builds = archive.getBuildRecords(build_state=NEEDS_BUILDING)

        logging.debug('Found %d pending builds.' % len(pending_builds))

        if len(pending_builds) == 0:

            logging.info('No more builds pending. We are done.')

            break

        used_capacity = get_archive_used_builders_capacity(archive)

        # For each processor, rescore up as many builds as we have

        # capacity for.

        for processor in args.processors:

            builds_to_rescore = (

                capacity[processor] - used_capacity.get(processor, 0))

            logging.debug(

                'Will try to rescore %d %s builds', builds_to_rescore,

                processor)

            for build in pending_builds:

                if builds_to_rescore <= 0:

                    break

                if build.arch_tag != processor:

                    continue

                if build.score < 0:

                    # Only rescore builds that look like the negative

                    # copy archive modified have been applied.

                    logging.info('Rescoring %s' % build.title)

                    # This should make them considered like a regular build.

                    build.rescore(

                        score=build.score + COPY_ARCHIVE_SCORE_PENALTY)

                else:

                    logging.debug('%s already rescored', build.title)

                # If the score was already above 0, it was probably

                # rescored already, count it against our limit anyway.

                builds_to_rescore -= 1

        # Reschedule in a while.

        logging.debug('Sleeping for %d minutes.', SCHEDULE_PERIOD)

        time.sleep(SCHEDULE_PERIOD * 60)

if __name__ == '__main__':

    main()