~abentley/ci-director/building-trumps-result

#!/usr/bin/env python
from argparse import ArgumentParser
from collections import namedtuple
from contextlib import contextmanager
from datetime import (
    datetime,
    timedelta,
)
from email.message import Message
import logging
from logging.handlers import RotatingFileHandler
import os
import os.path
import random
import re
import shlex
from smtplib import SMTP
from socket import setdefaulttimeout
import sys
from time import sleep
import urllib2

from jenkins import (
    Jenkins,
    JenkinsException,
    NotFoundException,
)
import yaml

from git import GitRepo
from jobs import (
    ABORTED,
    BUILD_REVISION,
    parse_tags,
    NOT_BUILT,
    PUBLISH_REVISION,
    REVISION_RESULTS,
    SUCCESS,
)
from storage import (
    BLESS,
    BUILDING,
    CloudHealthJob,
    CURSE,
    FAILED,
    FINAL_RESULTS,
    PENDING,
    SUCCEEDED,
    StateFile,
    StateFileInUse,
    TERMINAL_STATUS,
)
from utility import (
    log_exceptions,
    S3Storage,
    )


__metaclass__ = type


class NoSuchBranch(Exception):

    def __init__(self, repository, branch):
        super(Exception, self).__init__(
            'The repository "{}" has no branch "{}".'.format(repository,
                                                             branch))


def get_build_parameters(build_info):
    """Return the build parameters as a dict."""
    parameters = {}
    for action in build_info.get('actions', []):
        parameter_list = action.get('parameters', [])
        parameters.update((p['name'], p['value']) for p in parameter_list)
    return parameters


def get_buildvars_file(jenkins, build_info):
    """Get the buildvars for a given build's buildvar file.

    If the build is building, the workspace version is retrieved.  Otherwise,
    the archived version is retrieved.

    :return: the text of the buildvars.bash file or None if it could not be
        retrieved.
    """
    if build_info['building']:
        url = '%sjob/build-revision/ws/buildvars.bash' % jenkins.server
    else:
        url = '%sjob/build-revision/%d/artifact/buildvars.bash' % (
            jenkins.server, build_info['number'])
    try:
        return jenkins.jenkins_open(urllib2.Request(url))
    except NotFoundException:
        return None


def parse_buildvars_file(buildvars):
    """Return a dict containing the assignments given in buildvars.bash.

    :param buildvars: The buildvars file as a string, i.e. a series of "export
        X=Y" lines, with optional quoting.
    :return: a dict of the assignments.
    """
    return dict(shlex.split(l)[1].split('=', 1)
                for l in buildvars.splitlines())


class ConfigReader:
    """Read the config file."""

    @staticmethod
    def config_file_name():
        """Return the config filename for the current user."""
        return os.path.join(os.environ['HOME'], '.config/ci-director')

    @classmethod
    def read_config(cls):
        """Read the config file for the current user."""
        with open(cls.config_file_name()) as config_file:
            return yaml.safe_load(config_file)


class ServerInfo:
    """Wrapper around the server info returned by Jenkins."""

    def __init__(self, info):
        self.info = info

    def get_building_jobs(self):
        """List all jobs which are currently building."""
        return [j['name'] for j in self.info['jobs']
                if j['color'].endswith('_anime')]

    def get_enabled_jobs(self):
        """List all jobs which are enabled on this Jenkins instance."""
        return [j['name'] for j in self.info['jobs']
                if not j['color'] == 'disabled']


class JobInfo:
    """Wrapper around the Job info returned by Jenkins."""

    def __init__(self, info):
        self.info = info

    @classmethod
    def from_jenkins(cls, jenkins, name):
        """Factory.  Return a JobInfo for this Jenkins and name.

        :param jenkins: a jenkins.Jenkins instance.
        :param name: The job name.
        :return: a JobInfo.
        """
        return cls(jenkins.get_job_info(name))

    @property
    def current_build_number(self):
        if self.info['lastBuild'] is None:
            return None
        return self.info['lastBuild']['number']

    @property
    def description(self):
        return self.info.get('description')


class Job:
    """Base class for jobs."""

    def __init__(self, logger, state_file, jenkins, vote=True):
        self.logger = logger
        self.state_file = state_file
        self.jenkins = jenkins
        self.vote = vote

    @classmethod
    def _get_available_jobs(cls, pattern, server_info, jenkins, state_file,
                            logger):
        for job_id in server_info.get_enabled_jobs():
            if not pattern.match(job_id):
                continue
            yield cls(job_id, logger, state_file, jenkins)

    @staticmethod
    def make_subclass(job_id, logger, state_file, jenkins):
        """Return an instance of the subclass matching this job_id.

        :return: None if no corresponding subclass, otherwise an instance.
        """
        job = ResourcefulJob.get_job_from_jenkins(
            job_id, jenkins, state_file, logger)
        if job is not None:
            return job
        return None

    def adopt_build(self, build_info):
        """Record build if unknown and against the current revision build.

        :param build_info: Info about the build to adopt.
        """
        params = get_build_parameters(build_info)
        if params.get('revision_build') != str(
                self.state_file.current_version_number):
            return
        build_number = build_info['number']
        job = self.get_state_file_job()
        if build_number > job.build_number:
            self.logger.info('Adopting %s #%d.', self.job_id, build_number)
            job.start_build(build_number)

    def build(self, parameters=None):
        """Build this job.

        :param parameters: a dict of parameters to supply when building this
            job.
        """
        self.logger.info('Building %s', self.job_id)
        self.jenkins.build_job(self.job_id, parameters)

    def conflicting_jobs(self, active_jobs):
        """Return True if any active jobs conflict with this one.

        Building a new revision conflicts with basically all other job types,
        because it changes the current version.

        :param active_jobs: A list of job_ids of building or queued jobs.
        """
        return bool(BUILD_REVISION in active_jobs)

    def maybe_build(self, active_jobs):
        """Build this job if appropriate and update state file.

        Will be built if
        - the revision build succeeded
        - there are no conflicting jobs
        - this job's StateFile entry has not reached a terminal status.
        :param active_jobs: The names of any jobs that are active (building or
            queued).
        :return: True if a build was requested, False if no build was
            requested.
        """
        if self.conflicting_jobs(active_jobs):
            return False
        for prereq in self.get_prerequisite_jobs():
            if prereq.test_id in active_jobs:
                return False
            if prereq.get_status() != SUCCEEDED:
                return False
        state_file_job = self.get_state_file_job()
        if state_file_job.get_status() in TERMINAL_STATUS:
            return False
        parameters = self.get_build_parameters()
        self.build(parameters)
        return True


def test_jobs_are_running(jenkins, active_jobs):
    if PUBLISH_REVISION in active_jobs:
        return True
    if REVISION_RESULTS in active_jobs:
        return True
    for job in active_jobs:
        found = ResourcefulJob.get_job_from_jenkins(
            job, jenkins, StateFile(), None)
        if found is not None:
            return True
    return False


class BuildRevisionJob(Job):
    """Build the packaging for a revision."""

    job_id = BUILD_REVISION

    def __init__(self, repo_path, branches, logger, state_file, jenkins):
        super(BuildRevisionJob, self).__init__(logger, state_file, jenkins)
        self.repo_path = repo_path
        self.branches = branches

    def conflicting_jobs(self, active_jobs):
        if test_jobs_are_running(self.jenkins, active_jobs):
            return True
        return super(BuildRevisionJob, self).conflicting_jobs(active_jobs)

    def maybe_build(self, active_jobs):
        """Prepare this revision if appropriate and update state file.

        Will be built if
        - No jobs of its type are active
        - No test builds are active
        :param active_jobs: The names of any jobs that are active (building or
            queued).
        :return: True if a build was requested, False if no build was
            requested.
        """
        if self.conflicting_jobs(active_jobs):
            return False
        branch, revision_id = self.get_revision_to_build()
        if revision_id is None:
            self.logger.info('No revision ready to build.')
            return False
        branch = 'gitbranch:{}:{}'.format(branch, self.repo_path)
        self.build({'branch': branch, 'revision': revision_id})
        return True

    @staticmethod
    def _extract_version(v_bytes):
        """Extract a version string from the contents of version.go."""
        return re.search('^const version = "(.*)"', v_bytes, re.M).group(1)

    def parse_branch_url(self, branch_url):
        """Extract branch name from a gitbranch url."""
        if not branch_url.startswith('gitbranch:'):
            raise ValueError('Unsupported branch type')
        branch, repository = branch_url[len('gitbranch:'):].split(':', 1)
        if repository != self.repo_path:
            raise ValueError(
                'Branch has wrong repository: {}'.format(branch_url))
        return branch

    def get_revision_to_build(self):
        """Find a revision to build in the git repository.

        :return: a tuple of branch, commit or None, None if no revisions need
            to be prepared.
        """
        priority_branches = [self.parse_branch_url(b)
                             for b in self.branches]
        repo = GitRepo('git://{}'.format(self.repo_path))
        tips = dict(repo.list_remote_branch_tips())
        branch_order = priority_branches
        branch_order.extend(sorted(set(tips).difference(branch_order)))
        for branch in branch_order:
            try:
                commit = tips[branch]
            except KeyError:
                raise NoSuchBranch(self.repo_path, branch)
            if self.state_file.revision_prepared(commit):
                self.logger.info('Skipping %s (%s)', branch, commit)
                continue
            return branch, commit
        return None, None

    def adopt_build(self, build_info):
        """Record this build-revision job using start_revision.

        The build is only adopted if newer than the current version.
        Data about the build is taken from buildvars.bash.
        """
        if build_info['number'] <= self.state_file.current_version_number:
            return
        buildvars_file = get_buildvars_file(self.jenkins, build_info)
        if buildvars_file is None:
            self.logger.info('Could not retrieve buildvars for %s' %
                             build_info['number'])
            return
        self.logger.info(
            'Adopting %s #%s' % (self.job_id, build_info['number']))
        buildvars = parse_buildvars_file(buildvars_file)
        revno = buildvars.get('REVNO')
        if revno is not None:
            revno = int(revno)
        self.state_file.start_revision(
            buildvars['BRANCH'], revno, buildvars['REVISION_ID'],
            buildvars['VERSION'], build_info['number'])


class ResourcefulJob(Job):
    """A job configured in Jenkins with its required resources."""

    def __init__(self, job_id, logger, state_file, jenkins,
                 requires=None, conflicts=None, group_name=None,
                 failure_threshold=1, tags=None, vote=True,
                 supported_substrates=None):
        super(ResourcefulJob, self).__init__(logger, state_file, jenkins)
        self.job_id = job_id
        self.requires = set(requires.split()) if requires else set([])
        self.conflicts = set(conflicts.split()) if conflicts else set([])
        self.group_name = group_name
        self.tags = tags
        self.failure_threshold = failure_threshold
        self.vote = vote
        self.supported_substrates = supported_substrates

    @staticmethod
    def config_from_description(job_id, description, logger,
                                section='ci-director'):
        """return a config dict parsed from the INI-ish text."""
        if not description:
            return None
        found_stanza = False
        config = {}
        options = (
            'requires', 'conflicts', 'group_name', 'failure_threshold', 'tags',
            'vote', 'supported_substrates')
        for line in description.splitlines():
            if not found_stanza and '[{}]'.format(section) == line.strip():
                found_stanza = True
            if found_stanza and ':' in line:
                key, value = line.split(':')
                key = key.strip().lower().replace('-', '_')
                if key in options:
                    value = value.strip().lower()
                    if key == 'failure_threshold':
                        try:
                            value = int(value)
                        except ValueError:
                            msg = 'Could not parse %s failure-threshold: %s'
                            logger.warning(msg, job_id, value)
                            value = 1
                    if key == 'vote':
                        value = False if value.lower() == 'false' else True
                    if key == 'supported_substrates':
                        value = [v.strip() for v in value.split(',')]
                    config[key] = value
        if found_stanza:
            return config
        return None

    @classmethod
    def get_available_jobs(cls, server_info, jenkins, state_file, logger):
        """Return an iterator of ResourcefulJobs.

        :param server_info: ServerInfo for the current Jenkins.
        :param jenkins: The Jenkins instance to be used by ResourcefulJob.
        :param state_file: The StateFile to be used by ResourcefulJob.
        :param logger: The logger to be used by ResourcefulJob.
        """
        for job_id in server_info.get_enabled_jobs():
            job = cls.get_job_from_jenkins(job_id, jenkins, state_file, logger)
            if job:
                yield job

    @classmethod
    def get_job_from_jenkins(cls, job_id, jenkins, state_file, logger):
        try:
            description = JobInfo.from_jenkins(jenkins, job_id).description
        except JenkinsException:
            # The job was renamed while CID was running.
            return None
        except Exception as e:
            logger.warn(
                'failed: JobInfo.from_jenkins(jenkins, {})'.format(job_id))
            logger.warn(str(e))
            return None
        config = cls.config_from_description(job_id, description, logger)
        if config:
            return cls(job_id, logger, state_file, jenkins, **config)
        return None

    def conflicting_jobs(self, active_job_ids):
        """Do any of the active jobs ids conflict with this job?"""
        if self.job_id in active_job_ids:
            return True
        if self.conflicts.intersection(set(active_job_ids)):
            return True
        return super(ResourcefulJob, self).conflicting_jobs(active_job_ids)

    def get_prerequisite_jobs(self):
        """The jobs required by this job to have succeeded.

        A job may be None if when it is missing, possibly disabled in jenkins.
        """
        required_jobs = {id: None for id in self.requires
                         if id != 'build-revision'}
        required_jobs['build'] = self.state_file.get_build_revision_job()
        for job in self.state_file.iter_grouped_jobs():
            if job.test_id in required_jobs:
                required_jobs[job.test_id] = job
        for job_id, job in required_jobs.items():
            if job is None:
                self.logger.warning(
                    '%s requires %s but it is missing', self.job_id, job_id)
        return required_jobs.values()

    def get_state_file_job(self):
        """Return the StateFileJob that corresponds to this job."""
        return self.state_file.get_grouped_job(
            self.job_id, self.group_name, self.failure_threshold, self.tags,
            self.vote,
            prerequisite_jobs=[BUILD_REVISION] + list(self.requires))

    def get_build_parameters(self):
        """Return the parameters for building, as a dict.

        If supported_substrates is non-None, a substrate will be chosen at
        random.  If a substrate is unhealthy, it will be avoided.  If all
        substrates are unhealthy, an unhealthy one will be used.
        """
        parameters = {'revision_build': self.state_file.current_version_number}
        if self.supported_substrates is not None:
            since = datetime.utcnow() - timedelta(hours=1)
            status = self.state_file.get_substrate_status(
                self.state_file.current_version_number,
                self.supported_substrates, since)
            substrates = [s for s in self.supported_substrates
                          if status.get(s, True)]
            # If no healthy substrates, pick an unhealthy one
            if len(substrates) == 0:
                substrates = self.supported_substrates
            parameters['substrate'] = random.choice(substrates)
        return parameters

    def maybe_build(self, active_jobs):
        """Run this job if appropriate.

        The Job will be run when
        - There are no conflicting jobs.
        - The required prerequisite jobs are present and have succeeded.
        - The StateFile considers the job status to be None or Pending.

        :return: True if a build was requested, otherwise False.
        """
        if self.conflicting_jobs(active_jobs):
            return False
        for prereq in self.get_prerequisite_jobs():
            if prereq is None:
                return False
            if prereq.test_id in active_jobs:
                return False
            if prereq.get_status() != SUCCEEDED:
                return False
        job_status = self.get_state_file_job().get_status()
        if job_status is not None and job_status != PENDING:
            return False
        self.build(self.get_build_parameters())
        return True


class Mailer:
    """Send email about results."""

    def __init__(self, from_address, to_address, reports_url):
        self.from_address = from_address
        self.to_address = to_address
        self.reports_url = reports_url

    @classmethod
    def from_config(cls, config):
        """Return a Mailer initialized from the supplied config."""
        return cls(config['from_address'], config['to_address'],
                   config['reports_url'])

    def get_test_lines(self, build_number, tests):
        for test in sorted(tests):
            yield '{0} build #{1} '.format(*test)
            yield '{0}/releases/{1}/job/{2}/attempt/{3}\n'.format(
                self.reports_url, build_number, *test)

    def format_failure(self, summary, result):
        """Return the body of a failure message.

        :param summary: The information about the build, as given by
            StateFile.get_current_version_summary()
        :param result: A DetailedResult
        """
        ident_tmpl = (
            "Build: #{build_number} Revision: {branch} {short_revision}"
            " Version: {release_number}\n")
        ident = ident_tmpl.format(**summary)
        build_number = summary['build_number']
        ident += '{}/releases/{}\n'.format(
            self.reports_url, build_number)
        test_lines = []
        if result.voting_failures:
            test_lines.append('Failed tests\n')
            test_lines.extend(
                self.get_test_lines(build_number, result.voting_failures))
        if result.non_voting_failures:
            if len(test_lines) > 0:
                test_lines.append('\n')
            test_lines.append('Failed non-voting tests\n')
            test_lines.extend(
                self.get_test_lines(build_number, result.non_voting_failures))
        return ''.join([ident, '\n'] + test_lines)

    def get_failure_message(self, summary, result, final=False):
        """Return an email.Message representing a curse message.

        :param summary: The information about the build, as given by
            StateFile.get_current_version_summary()
        :param result: A DetailedResult
        :param final: If True, the failure is a final, comprehensive listing
            of the cursing failures.
        """
        message = Message()
        message['To'] = self.to_address
        message['From'] = 'CI & CD Jenkins <%s>' % self.from_address
        final_str = ' (final)' if final else ''
        lead_str = {CURSE: 'Cursed', BLESS: 'Blessed'}[result.result]
        subj_tmpl = '{0}{1}: #{build_number} {branch} {short_revision} '
        subj_summary = subj_tmpl.format(lead_str, final_str, **summary)
        tests = ', '.join(name for name, num in sorted(result.voting_failures))
        message['Subject'] = subj_summary + '(' + tests + ')'
        message.set_payload(self.format_failure(summary, result))
        return message

    def send_mail(self, message):
        """Send a message via SMTP."""
        smtp = SMTP()
        smtp.connect()
        smtp.sendmail(self.from_address, self.to_address, message.as_string())


DetailedResult = namedtuple('DetailedResult', ['result', 'voting_failures',
                                               'non_voting_failures'])


class JobSource:

    def __init__(self, logger, state_file, jenkins):
        self.logger = logger
        self.state_file = state_file
        self.jenkins = jenkins

    def get_candidate_determine_result_jobs(self):
        """Iterate through the StateFileJobs that may determine the result."""
        server_info = ServerInfo(self.jenkins.get_info())
        yield self.state_file.get_build_revision_job()
        resourceful_job_ids = []
        for job in ResourcefulJob.get_available_jobs(
                server_info, self.jenkins, self.state_file, self.logger):
            resourceful_job_ids.append(job.job_id)
            yield job.get_state_file_job()


class BaseResultJudge:
    """Class to determine build revision status.

    Uses a job source like JobSource to access jobs.
    """
    def __init__(self, job_source):
        self.job_source = job_source

    def get_candidate_determine_result_jobs(self):
        """Iterate through the jobs that may determine the result."""
        return self.job_source.get_candidate_determine_result_jobs()

    def result_tests_complete(self):
        """Return True if no new failures are expected.

        If the result is BLESS, this will happen at the same time, but this
        may happen significantly later than a CURSE, because the first failure
        causes a CURSE, but there may be many failures.
        """
        for job in self.get_candidate_determine_result_jobs():
            if (job.get_status() not in TERMINAL_STATUS and
                    not job.is_blocked()):
                return False
        return True

    def determine_result(self):
        """Return a DetailedResult describing the test result.

        If any voting builds have failed, the result is considered a CURSE
        Otherwise, if any voting builds have not voted, the result is
        considered PENDING.
        If all voting builds have voted and none failed, the result is
        considered a BLESS.

        Also included: a list of non_voting failures and a list of voting
        failures.
        """
        final = True
        voting_failures = []
        non_voting_failures = []
        for job in self.get_candidate_determine_result_jobs():
            build_tuple = (job.test_id, job.build_number)
            if not job.voting_build:
                if job.get_status() == FAILED:
                    non_voting_failures.append(build_tuple)
                continue
            if job.get_status() == FAILED:
                voting_failures.append(build_tuple)
            elif job.get_status() != SUCCEEDED:
                final = False
        if len(voting_failures) > 0:
            result = CURSE
        elif final:
            result = BLESS
        else:
            result = PENDING
        return DetailedResult(result=result, voting_failures=voting_failures,
                              non_voting_failures=non_voting_failures)


class ResultJudge(BaseResultJudge):
    """Class to determine build revision status and send mail.

    Uses a StateFile and Jenkins to access jobs.
    """

    def __init__(self, logger, state_file, jenkins):
        job_source = JobSource(logger, state_file, jenkins)
        super(ResultJudge, self).__init__(job_source)
        self.state_file = state_file

    def finalize(self, mailer):
        """Send an email when all tests are finished and when an error occured.

        Return True when all tests are finished and when the final result
        has been stored in the state file, else return False.
        """
        if not self.result_tests_complete():
            return False
        info = self.state_file.get_complete_failure_info(
            self.state_file.current_version_number)
        result = self.determine_result()
        if info is not None and dict(result.voting_failures) == info:
            return False
        if result.voting_failures or result.non_voting_failures:
            message = mailer.get_failure_message(
                self.state_file.get_current_version_summary(), result,
                final=True)
            mailer.send_mail(message)
        self.state_file.set_complete_failure_info(
            self.state_file.current_version_number,
            dict(result.voting_failures))
        return True


class RecordResultJob(Job):
    """"Record the final results for a revision."""

    job_id = REVISION_RESULTS

    def __init__(self, logger, state_file, jenkins, mailer=None):
        super(RecordResultJob, self).__init__(logger, state_file, jenkins)
        self.mailer = mailer
        self.judge = ResultJudge(logger, state_file, jenkins)

    def maybe_build(self):
        """Request a build to record the bless/curse results, if needed.

        Only BLESS and CURSE, not PENDING are recorded.
        If the results have already been recorded and have not changed, no
        build is done.
        """
        if self.state_file.current_version_number is None:
            return False
        result = self.judge.determine_result()
        self.logger.info('Final result is "%s".' % result.result)
        if result.result not in FINAL_RESULTS:
            return False
        if result.result == self.state_file.get_result()[0]:
            return False
        parameters = {'blessed': {BLESS: 'true', CURSE: 'false'}[
            result.result]}
        summary = self.state_file.get_current_version_summary()
        parameters.update(summary)
        build_number = self.build_get_number(parameters)
        self.state_file.save_result(result.result, build_number)
        if result.result == CURSE:
            message = self.mailer.get_failure_message(summary, result)
            self.mailer.send_mail(message)
        return True

    def build_get_number(self, parameters=None):
        """Build this job, and return its number.

        (Assumes job has a job_id member variable.)
        :param parameters: a dict of parameters to supply when building this
            job.
        :raises: an AssertionError if the number cannot be determined within 30
            seconds.
        """
        ji = JobInfo.from_jenkins(self.jenkins, self.job_id)
        old_build_number = ji.current_build_number
        self.build(parameters)
        for x in range(30):
            ji = JobInfo.from_jenkins(self.jenkins, self.job_id)
            build_number = ji.current_build_number
            if build_number != old_build_number:
                return build_number
            if build_number is None:
                self.logger.info('Waiting for first build.')
            else:
                self.logger.info('Stuck on %d' % build_number)
            sleep(1)
        else:
            raise AssertionError('Could not find build number.')


def make_jenkins(config):
    return Jenkins(config['jenkins_url'], config['jenkins_user'],
                   config['jenkins_password'])


class CIDirector:
    """Main control logic."""

    def __init__(self, jenkins, state_file=None, branches=None, mailer=None,
                 repo_path=None):
        self.jenkins = jenkins
        self.state_file = state_file
        if branches is None:
            branches = []
        self.branches = branches
        self.logger = logging.getLogger('cidirector')
        self.mailer = mailer
        self.repo_path = repo_path

    @classmethod
    def from_config(cls, config, state_file, branches):
        """Create an instace from a config, StateFile and list of branches."""
        mailer = Mailer.from_config(config)
        return CIDirector(
            make_jenkins(config),
            state_file, branches, mailer, config['repo_path'])

    @classmethod
    @contextmanager
    def stateful(cls, branches, save=True):
        """Provide a CIDirector whose state will be saved to disk.

        :param branches: The branches that the CIDirector should check for new
            revisions.
        :param save: Determine whether the state will be saved to disk (False
            is useful for testing.)
        """
        config = ConfigReader.read_config()
        with StateFile.acquire(S3Storage.from_config(config)) as (state_file,
                                                                  fileobj):
            director = cls.from_config(config, state_file, branches)
            try:
                yield director
            finally:
                if save:
                    state_file.write()

    def current_version_active(self):
        """Determine whether the current build is the jenkins build."""
        job_info = JobInfo.from_jenkins(self.jenkins, BUILD_REVISION)
        if (job_info.current_build_number !=
                self.state_file.current_version_number):
            return False
        else:
            return True

    def current_version_published(self):
        """Determine whether the current version is published.

        It is not ready if no publication job has successfully completed.
        :param build_number: build number of prepare-new-version.
        :return: True if ready, False if not.
        """
        pub_job = self.state_file.publication_job()
        if pub_job is None:
            return False
        if not pub_job.get_status() == SUCCEEDED:
            return False
        return True

    def get_active_jobs(self, server_info):
        """Return a list of active jobs, including both building and queued.

        :param server_info: A ServerInfo to use for determining building jobs.
        :return: a list of job names.
        """
        active = server_info.get_building_jobs()
        queued = self.jenkins.get_queue_info()
        active.extend(item['task']['name'] for item in queued)
        return active

    def schedule_builds(self, server_info):
        """Request builds for appropriate jobs.

        Can schedule BuildRevisionJob, ResourcefulJob, RecordResultJob.
        :param server_info: ServerInfo for the current Jenkins.
        """
        self.adopt_builds(server_info)
        active_jobs = set(self.get_active_jobs(server_info))
        self.logger.info('Active jobs: %s', ', '.join(sorted(active_jobs)))
        candidate_jobs = []
        self.update_builds()
        judge = ResultJudge(self.logger, self.state_file, self.jenkins)
        all_results_ready = judge.finalize(self.mailer)
        job = RecordResultJob(
            self.logger, self.state_file, self.jenkins, self.mailer)
        judged = job.maybe_build()
        if all_results_ready or judged:
            self.state_file.finish_revision(datetime.utcnow())
            # Disabled in favour of update-outcomes.
            # self.state_file.build_info_to_s3()
        resourcefuljobs = ResourcefulJob.get_available_jobs(
            server_info, self.jenkins, self.state_file, self.logger)
        for r_job in resourcefuljobs:
                candidate_jobs.append(r_job)
        candidate_jobs.append(BuildRevisionJob(
            self.repo_path, self.branches, self.logger, self.state_file,
            self.jenkins))
        for job in candidate_jobs:
            if job.maybe_build(active_jobs):
                # We create a new set so that tests that access the parameters
                # of maybe_build don't get mutated results.
                active_jobs = active_jobs.union({job.job_id})
        if should_update_health(self.jenkins, active_jobs):
            jobs = list_cloud_health(server_info, self.jenkins, self.logger)
            self.update_cloud_health(jobs)

    def adopt_builds(self, server_info):
        """Adopt any builds for the build-revision not already recorded."""
        # We happen to know that branch will not be used here.
        job_instances = [BuildRevisionJob(self.repo_path, None, self.logger,
                         self.state_file, self.jenkins)]
        job_instances.extend(
            Job.make_subclass(si_job['name'], self.logger, self.state_file,
                              self.jenkins)
            for si_job in server_info.info['jobs'])
        for job in job_instances:
            if job is None:
                continue
            job_info = JobInfo.from_jenkins(self.jenkins, job.job_id)
            if job_info.current_build_number is None:
                continue
            build_info = self.jenkins.get_build_info(
                job.job_id, job_info.current_build_number)
            job.adopt_build(build_info)

    def update_builds(self):
        """Record the status of jobs that are not currently building."""
        self.logger.info('Updating job outcomes.')
        server_info = ServerInfo(self.jenkins.get_info())
        building = server_info.get_building_jobs()
        jobs = set([])
        current_version = self.state_file.current_version_number
        if current_version is not None:
            jobs.add(self.state_file.get_build_revision_job())
            for job in self.state_file.iter_grouped_jobs():
                jobs.add(job)
        for job in jobs:
            if job.test_id in building or job.get_status() != BUILDING:
                continue
            job.update_from_jenkins(self.jenkins, current_version,
                                    self.state_file)
            self.logger.info('Updated: %s', job)
        self.state_file.update_blocked()

    def update_cloud_health(self, jobs):
        version = self.state_file.current_version_number
        cloud_health = self.state_file.get_cloud_health(version)
        for job in jobs:
            substrate = cloud_health['substrates'].get(job.substrate, {})
            if job.last_completed in substrate.get(job.name, {}):
                continue
            info = self.jenkins.get_build_info(job.name, job.last_completed)
            result = info['result']
            if result in (ABORTED, NOT_BUILT):
                continue
            if result == SUCCESS:
                status = SUCCEEDED
            else:
                status = FAILED
            self.state_file.update_cloud_health(
                version, job, status, info['timestamp'], info['duration'])


def build_revision(branches):
    """Top-level logic for building jobs.

    For a supplied list of branches, it
    - Uses the config to access Jenkins, and loads the current state.
    - Runs jobs using CIDirector.
    - Saves the updated state.
    """
    with CIDirector.stateful(branches) as director:
        server_info = ServerInfo(director.jenkins.get_info())
        director.schedule_builds(server_info)


def list_cloud_health(si, jenkins, logger):
    """Iterate through CloudHealthJobs for every relevent job on Jenkins."""
    for job in si.info['jobs']:
        job_info = jenkins.get_job_info(job['name'])
        config = ResourcefulJob.config_from_description(
            job['name'], job_info['description'], logger,
            section='cloud-health')
        if config is None:
            continue
        tags = parse_tags(config.get('tags', ''))
        last_completed_num = job_info.get(
            'lastCompletedBuild', {}).get('number')
        yield CloudHealthJob(
            name=job['name'],
            last_completed=last_completed_num,
            substrate=tags['substrate'][0])


def should_update_health(jenkins, active_jobs):
    # Don't update cloud health when building, because current_version may be
    # out of date.
    if BUILD_REVISION in active_jobs:
        return False
    return test_jobs_are_running(jenkins, active_jobs)


def get_arg_parser():
    """Return the argument parser for this program."""
    parser = ArgumentParser('Trigger builds for release testing.')
    parser.add_argument('branch', help='The branches to give priority to.',
                        nargs='+')
    parser.add_argument('-v', '--verbose', help='Verbose info',
                        action='store_true')
    parser.add_argument('--log-path', help='Where to write logs.',
                        default='ci-director.log')
    parser.add_argument('--log-count', help='The number of backups to keep.',
                        default=2, type=int)
    return parser


def setup_logging(log_path, log_count, verbose=False):
    """Install log handers to output to file and stream."""
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
                                  '%Y-%m-%d %H:%M:%S')
    root_logger = logging.getLogger()
    rf_handler = RotatingFileHandler(
        log_path, maxBytes=1024 * 1024, backupCount=log_count)
    rf_handler.setFormatter(formatter)
    root_logger.addHandler(rf_handler)
    s_handler = logging.StreamHandler()
    s_handler.setFormatter(formatter)
    root_logger.addHandler(s_handler)
    if verbose:
        root_logger.setLevel(logging.INFO)


def main(argv=None):
    setdefaulttimeout(30)
    args = get_arg_parser().parse_args(argv)
    setup_logging(args.log_path, args.log_count, args.verbose)
    logger = logging.getLogger('cidirector')
    with log_exceptions(logger) as result:
        try:
            build_revision(args.branch)
        except StateFileInUse:
            logger.warning('State file already in use.')
            return 1
    return result.exit_status


if __name__ == '__main__':
    sys.exit(main())