~linaro-toolchain-dev/cbuild/tools

"""Script that monitors merge requests, branches unseen ones, and sets
up a build.
"""

import os
import os.path
import re
import subprocess
import json
import socket
import datetime
import logging
import getopt
import sys
import pprint
import pdb
import StringIO
import findlogs

import launchpadlib.launchpad

# Message sent when a merge request is snapshotted and queued for build
QUEUED = {
    'subject': "[cbuild] Queued %(snapshot)s for build",
    'body': """
cbuild has taken a snapshot of this branch at r%(revno)s and queued it for build.

The diff against the ancestor r%(ancestor)s is available at:
 http://cbuild.validation.linaro.org/snapshots/%(snapshot)s.diff

and will be built on the following builders:
 %(builders)s

You can track the build queue at:
 http://cbuild.validation.linaro.org/helpers/scheduler

cbuild-snapshot: %(snapshot)s
cbuild-ancestor: %(target)s+bzr%(ancestor)s
cbuild-state: check
"""
}

# Message sent when a individual build completes
CHECK_OK = {
    'subject': "[cbuild] Build OK for %(snapshot)s on %(build)s",
    'body': """
cbuild successfully built this on %(build)s.

The build results are available at:
 http://cbuild.validation.linaro.org/build/%(snapshot)s/logs/%(build)s

%(diff)s

The full testsuite results are at:
 http://cbuild.validation.linaro.org/build/%(snapshot)s/logs/%(build)s/gcc-testsuite.txt

cbuild-checked: %(build)s
"""
}

# Message sent when a individual build fails
CHECK_FAILED = {
    'subject': "[cbuild] Build failed for %(snapshot)s on %(build)s",
    'body': """
cbuild had trouble building this on %(build)s.
See the following failure logs:
 %(logs)s

under the build results at:
 http://cbuild.validation.linaro.org/build/%(snapshot)s/logs/%(build)s

%(diff)s

cbuild-checked: %(build)s
""",
    'vote': 'Needs Fixing',
    'nice': False
}

CONFIG = json.load(open('taker.json'))

def run(cmd, cwd=None, const=False):
    logging.debug('Executing "%s" (cwd=%s)' % (' '.join(cmd), cwd))

    if const or not get_config2(False, 'dry-run'):
        child = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE)
        stdout, stderr = child.communicate()
        retcode = child.wait()

        if retcode:
            raise Exception('Child process returned %s' % retcode)

        lines = [x.rstrip() for x in StringIO.StringIO(stdout).readlines()]
        logging.debug('Stdout: %r' % lines[:20])

        return lines
    else:
        return []

def tidy_branch(v):
    """Tidy up a branch name from one of the staging servers to the
    top level server.
    """
    return v.replace('lp://qastaging/', 'lp:')

def get_config_raw(v):
    at = CONFIG

    for name in v.split('!'):
        at = at[name]

    return at

def get_config(*args):
    v = '!'.join(args)

    # See if the value exists in the hosts override first
    try:
        return get_config_raw('hosts!%s!%s' % (socket.gethostname(), v))
    except KeyError:
        return get_config_raw(v)

def get_config2(default, *args):
    """Get a configuration value, falling back to the default if not set."""
    try:
        return get_config(*args)
    except KeyError:
        return default

class Proposal:
    def __init__(self, proposal):
        self.proposal = proposal
        self.parse(proposal)

    def parse(self, proposal):
        """Parse the proposal by scanning all comments for
        'cbuild-foo: bar' tokens.  Pull these into member variables.
        """
        self.state = None
        self.ancestor = None
        self.checked = []

        for comment in proposal.all_comments:
            lines = [x.strip() for x in comment.message_body.split('\n')]

            for line in lines:
                match = re.match('cbuild-(\w+):\s+(\S+)', line)

                if match:
                    name, value = match.groups()

                    if name in ['checked']:
                        getattr(self, name).append(value)
                    else:
                        setattr(self, name, value)

def add_comment(proposal, template, **kwargs):
    """Add a comment using the given template for the subject and
    body text.
    """
    vote = template.get('vote', None)
    # True if this is a 'happy' result.  Used in debugging to
    # prevent unreversable results from going up to Launchpad.
    nice = template.get('nice', True)

    subject = (template['subject'] % kwargs).rstrip()
    body = (template['body'] % kwargs).rstrip()

    just_print = get_config2(False, 'dry-run')

    if not nice and get_config2(True, 'cautious'):
        # Running in a debug/test mode.  Don't push unreversable
        # results.
        just_print = True

    if get_config2(False, 'prompt'):
        logging.info("Want to add this comment: %s\n%s\n\nVote: %s\n" % (subject, body, vote))

        if just_print:
            got = raw_input('Add it anyway? > ')
        else:
            got = raw_input('OK to add? > ')

        just_print = got.strip() not in ['Y', 'y']

    if just_print:
        logging.info("Would have added this comment: %s\n%s\n\nVote: %s\n" % (subject, body, vote))
    else:
        if vote:
            proposal.proposal.createComment(subject=subject, content=body, vote=vote)
        else:
            proposal.proposal.createComment(subject=subject, content=body)

def queue(proposal):
    """Run the queue step by snapshoting and queuing the build."""
    lp = proposal.proposal

    source = lp.source_branch
    owner = source.owner.name
    branch = tidy_branch(source.bzr_identity)
    revno = source.revision_count

    # We might have hit this before Launchpad has scanned it
    if not revno:
        # Work around Launchpad being broken on 2011-09-07 by assuming
        # the merge was against the latest revno
        # Work around bzr revno bug with ghost ancestry
        # (https://launchpad.net/bugs/1161018)
        revno = run(['bzr', 'revno', 'nosmart+%s' % branch], const=True)
        revno = int(revno[0])
        assert revno

    target = tidy_branch(lp.target_branch.bzr_identity)

    # Find where to check it out
    prefix = get_config('branches', target, 'prefix')
    reporoot = get_config('branches', target, 'reporoot')

    # Give it an archive name
    suffix = '~%s~%s' % (owner, os.path.basename(branch))

    snapshot = '%s+bzr%s%s' % (prefix, revno, suffix)
    path = '%s/%s/%s' % (get_config('repos'), reporoot, snapshot)

    # Remove the old directory
    run(['rm', '-rf', path])
    # Branch it
    run(['bzr', 'branch', '--no-tree', branch, path], cwd=os.path.dirname(path))
    # Find the common ancestor
    lines = run(['bzr', 'revision-info', '-d', path, '-r', 'ancestor:%s' % target])

    ancestor = lines[0].split()[0] if lines else 'unknown'

    # Make the snapshot
    lines = run(['%s/pull_branch.sh' % get_config('tools'), path, prefix, '%s' % revno, suffix, target])

    # Scan the tool results to see which queues the build was
    # pushed into
    builders = []

    for line in lines:
        match = re.match('Spawned into (\S+)', line)
        
        if match:
            builders.append(match.group(1))

    add_comment(proposal, QUEUED, snapshot=snapshot, revno=revno, builders=' '.join(builders), ancestor=ancestor, target=target)

def make_diff(proposal, all_files, check):
    """Check if there are any DejaGNU summary files and diff them."""
    sums = [x for x in check if '.sum' in x[-1]]

    if not proposal.ancestor:
        lines = ['The test suite was not checked as the branch point was not recorded.']
    elif not sums:
        lines = ['The test suite was not checked as this build has no .sum style test results']
    else:
        # Have some test results.  See if there's a corresponding
        # version in the branch point build.
        first = '/'.join(sums[0])
        ref = findlogs.find(all_files, first, proposal.ancestor)

        if not ref:
            lines = ['The test suite was not checked as the branch point %s has nothing to compare against.' % proposal.ancestor]
        else:
            revision, build = ref

            # Generate a unified diff between the two sets of results
            diff = run(['%s/difftests.sh' % get_config('tools'),
                        '%s/%s/logs/%s' % (get_config('build'), revision, build),
                        '%s/%s' % (get_config('build'), os.path.dirname(first))],
                       const=True)                           

            if diff:
                lines = ['The test suite results changed compared to the branch point %s:' % proposal.ancestor]
                lines.extend([' %s' % x for x in diff])
            else:
                lines = ['The test suite results were unchanged compared to the branch point %s.' % proposal.ancestor]

    if len(lines) > 40:
        total = len(lines)
        lines = lines[:40]
        lines.append(' ...and %d more' % (total - len(lines)))
            
    return '\n'.join(lines)

def check_build(proposal, all_files, build, matches):
    """Check a single build to see how the results went."""

    if build in proposal.checked:
        logging.debug('Already posted a comment on %s' % build)
    else:
        check = [x for x in matches if len(x) >= 4 and x[2] == build]
        failures = [x for x in check if 'failed' in x[-1]]
        finished = 'finished.txt' in [x[-1] for x in check]

        logging.info('Checking %s (finished=%s, failures=%r)' % (build, finished, failures))

        if finished:
            logs = ' '.join(x[-1] for x in failures)

            diff = make_diff(proposal, all_files, check)

            if failures:
                add_comment(proposal, CHECK_FAILED, build=build, snapshot=proposal.snapshot, logs=logs, diff=diff)
            else:
                add_comment(proposal, CHECK_OK, build=build, snapshot=proposal.snapshot, logs=logs, diff=diff)

def check(proposal, all_files):
    """Check an already queued build for build results."""
    # Find all log files for this snapshot
    matches = [x for x in all_files if x[0] == proposal.snapshot]
    matches = [x for x in matches if len(x) >= 3 and x[1] == 'logs']

    if not matches:
        logging.debug('No log files yet')
        return

    # Build a list of builds
    builds = [x[2] for x in matches if len(x) == 3]

    for build in builds:
        check_build(proposal, all_files, build, matches)

def run_proposal(proposal, all_files, allowed):
    when = proposal.date_review_requested

    # Only run recent proposals that have had review requested
    if when:
        now = datetime.datetime.now(when.tzinfo)
        elapsed = (now - when).days

        logging.info('Checking proposal %s which is %d days old' % (proposal, elapsed))

        if elapsed <= get_config2(14, 'age-limit'):
            p = Proposal(proposal)

            state = p.state if p.state else 'queue'

            if state in allowed:
                # Could use a dodgy getattr()...
                if state == 'queue':
                    queue(p)
                elif state == 'check':
                    check(p, all_files)
                else:
                    assert False, 'Proposal %s is in the invalid state "%s"' % (proposal, p.state)
            else:
                logging.info("Skipping %s" % state)

def main():
    opts, args = getopt.getopt(sys.argv[1:], 'f:vs:')
    verbose = int(get_config2(0, 'verbose'))
    steps = get_config2('queue', 'steps').split(',')

    for opt, arg in opts:
        if opt == '-f':
            name, value = arg.split('=')
            option = json.loads('{ "%s": %s }' % (name, value))
            CONFIG.update(option)
        elif opt == '-v':
            verbose += 1
        elif opt == '-s':
            steps = arg.split(',')

    if verbose >= 2:
        logging.basicConfig(level=logging.DEBUG)
    elif verbose >= 1:
        logging.basicConfig(level=logging.INFO)

    # Parse all-files into a list of already split paths
    with open(get_config('all-files')) as f:
        all_files = f.readlines()

    all_files = [x.rstrip().split('/') for x in all_files]
    all_files = [x[1:] for x in all_files if len(x) >= 2]

    # Login to Launchpad
    launchpad = launchpadlib.launchpad.Launchpad.login_with('taker', get_config('instance'), credentials_file=os.path.expanduser('~/.launchpadlib/credentials'))

    # Scan all projects...
    for name in get_config('projects'):
        logging.info('Checking project %s' % name)
        project = launchpad.projects[name]
         
        # Scan all proposals in the project...
        proposals = project.getMergeProposals()

        for p in proposals:
            try:
                run_proposal(p, all_files, steps)
            except Exception, ex:
                logging.error('Error while processing %s: %s' % (p, ex))

if __name__ == '__main__':
    main()