~michaelh1/+junk/tcwg-web

#!/usr/bin/env python
import pickle
import re
import os
import collections
import socket
import pprint
import glob

import web
import web.webopenid

import launchpadlib.launchpad

import gcchelpers

urls = (
  '/helpers/patchtrack', 'patchtrack',
  '/helpers/patchtrack/create/(.*)', 'patchtrackcreate',
  '/helpers/testlog/(.*)', 'testlog',
  '/helpers/buildlog', 'buildlog',
  '/helpers/testcompare/(.*)', 'testcompare',
  '/helpers/tickets', 'tickets',
  '/helpers/host', 'web.webopenid.host',
  '/helpers/login', 'login',
)

def ctx_processor(handle):
    # do whatever you need to here
    web.ctx.oid = web.webopenid.status()
    web.ctx.user = None

    if web.ctx.oid:
        match = re.match(r'https://launchpad.net/~(.+)', web.ctx.oid)

        if match:
            web.ctx.user = match.group(1)

    web.ctx.msg = web.input().get('msg', None)

    return handle()

app = web.application(urls, globals())
render = web.template.render('templates/', base='layout', globals={'ctx': web.ctx})
# Add the processor
app.add_processor(ctx_processor)

def shorten(v):
    if '. ' in v:
        v = v[:v.index('. ')+1]

    if '\n' in v:
        v = v[:v.index('\n')]

    limit = 200

    if len(v) > limit:
        v = v[:limit] + '...'

    return v

def load_db():
    lp = pickle.load(open('lpcache.pickle', 'rb'))['lp']
    bzr = pickle.load(open('bzrcache.pickle', 'rb'))

    person_map = {}

    for name, value in lp.person.items():
        person_map[value.display_name] = value

    revision_map = {}

    # Drop any bugs tagged with 'tcwg-ignore'
    ignore = [x for x in lp.bugs if 'tcwg-ignore' in lp.bugs[x].tags]

    # Override for now
    ignore = []

    for i in ignore:
        del lp.bugs[i]

    for name, db in bzr['branches'].items():
        revisions = db['revisions']

        for revision in revisions:
            # Give the revision a short name
            revision.short = shorten(revision.message)
            # Record the fully qualified name
            fqn = '%s,revno=%d' % (name, revision.revno)
            revision_map[fqn] = revision

            # Try to map to a user
            mapped_authors = []

            for author in revision.authors:
                match = re.match('(.+\S)\s+<.+>', author)

                if match and match.group(1) in person_map:
                    mapped = person_map[match.group(1)]
                else:
                    mapped = person_map.get(name, None)

                if mapped:
                    mapped_authors.append(mapped)
            
            revision.mapped_authors = mapped_authors

            # Cross-reference
            revision.bugs = []

            for id in revision.fixes:
                if id in lp.bugs:
                    revision.bugs.append(lp.bugs[id])

    project = lp.projects['gcc-linaro']

    for task in project.tasks:
        for content in [task.bug.description] + [x.content for x in task.bug.messages]:
            # Scan all messages for commands to tie revisions and tickets together
            for line in content.split('\n'):
                rev = None

                match = re.match('[Rr]elated:\s+(\S+)\s*$', line)

                if match:
                    rev = match.group(1)
                else:
                    match = re.match('fixed-in:\s+(.+)$', line)

                    if match:
                        rev = match.group(1)

                if rev:
                    print rev

                    if rev in revision_map:
                        fqn = rev
                    elif rev.isdigit():
                        # Convert old links with an implicit 4.4 reference into a
                        # fully qualified name
                        fqn = 'lp:gcc-linaro/4.4,revno=%s' % rev

                    if fqn in revision_map:
                        if task.bug not in revision_map[fqn].bugs:
                            revision_map[fqn].bugs.append(task.bug)
                    else:
                        print 'Unrecogniesed revision %s on %s' % (rev, line)

    # Add short names to all bugs
    for bug in lp.bugs.values():
        bug.short = shorten(bug.title)

    # Add links into Launchpad for all items
    for item in lp.by_self_link.values():
        # Convert a https://api.launchpad.net/1.0/foo link to https://launchpad.net/foo
        item.lp_link = re.sub(r'(.+)\://api\.(.+)/1\.0/(.+)', r'\1://\2/\3', item.self_link)

    db = {
        'lp': lp,
        'project': project,
        'branches': bzr['branches'],
        'revision_map': revision_map,
        }

    return db

def create_lp():
     cachedir = os.path.expanduser('~/.tcwg-web/cache/')
     lp = launchpadlib.launchpad.Launchpad.login_with('tcwg-web', 'production', cachedir)

     return lp

def authenticate():
    oid = web.webopenid.status()

    if not oid:
        raise web.seeother('/helpers/login?msg=Login required')

    match = re.match(r'https://launchpad.net/~(.+)', oid)

    if not match:
        raise web.seeother('/helpers/login?msg=Not a Launchpad account')

    user = match.group(1)

    if user not in ['michaelh1', 'ams-codesourcery']:
        raise web.seeother('/helpers/login?msg=Permission denied')

    return user

def host_name():
    return socket.gethostname()

def running_on(name):
    return host_name() == name

def map_filename(name):
    if running_on('crucis'):
        base = '/home/michaelh/+junk/ex/build'
    else:
        base = '/var/www/ex.seabright.co.nz/build'

    if not name:
        return None

    abspath = os.path.abspath(os.path.join(base, name))

    if not abspath.startswith(base):
        return None
    else:
        return abspath

class patchtrack:
    def GET(self):
        db = load_db()
        return render.patchtrack(db)

class Line:
    def __init__(self, lineno, text, type, ticket=None):
        self.lineno = lineno
        self.text = text
        self.type = type
        self.ticket = ticket

ticketmap = {
    'g++.dg/eh/pr42859.C': 602168,
    'g++.dg/vect/pr36648.cc': 602186,
    'gcc.dg/autopar/*': 602190,
    'gcc.dg/Warray-bounds-3.c': 602277,
    'gcc.dg/tree-ssa/predcom*': 602285,
    'gcc.dg/vect/*': 602287,
    'gcc.target/arm/vfp-ldmias.c': 602288,
    'gcc.target/arm/vfp-stmias.c': 602288,
    'gfortran.dg/vect/*': 602291,
    'gcc.dg/tree-prof/*': 612402,
    'gcc.c-torture/execute/990208-1.c': 612405,
    'gcc.c-torture/execute/bcp-1.c': 612406,
}

def map_ticket(line):
    match = re.match('[A-Z]+: (\S+)', line)

    if match:
        test = match.group(1)

        for name, ticket in ticketmap.items():
            if name.endswith('*'):
                if test.startswith(name[:-1]):
                    return ticket
            elif name == test:
                return ticket

    return None

class testlog:
    def GET(self, log):
        with open('/var/www/ex.seabright.co.nz/build/' + log) as f:
            lines = f.readlines()

        post = []

        for i, line in enumerate(lines):
            line = line.rstrip()

            if 'FAIL:' in line:
                type = 'fail'
            elif 'Running ' in line:
                type = 'note'
            elif '=== ' in line:
                type = 'header'
            elif line.startswith('# of '):
                type = 'note'
            elif re.match('[A-Z]+: ', line):
                type = 'mark'
            else:
                type = 'other'

            post.append(Line(i+1, line, type, map_ticket(line)))
        
        return render.testlog(log, post)

def find_builds():
    root = map_filename('.')
    hosts = {}
    builds = {}
    cpus = {}

    drop, dirnames, drop = os.walk(root).next()

    for name in dirnames:
        builds[name] = {}

        drop, drop, filenames = os.walk(os.path.join(root, name)).next()
        
        for fname in filenames:
            if fname.endswith('.tar.xz'):
                parts = fname.split('.')[-3].split('-')
                
                if len(parts) > 4:
                    cpu, distro, ver, host = parts[-4:]

                    hosts[host] = cpu
                    cpus[cpu] = True

                    logs = '%s/logs/%s-%s-%s-%s' % (name, cpu, distro, ver, host)

                    languages = None

                    try:
                        with open(root + logs + '/gcc-configure.txt') as f:
                            for line in f.readlines():
                                line = line.strip()
                                
                                if line.startswith('The following'):
                                    end = line.split(': ')[-1]
                                    languages = end.split(',')
                    except Exception, ex:
                        print '!!', ex

                    builds[name][host] = {
                        'binary': '%s/%s' % (name, fname),
                        'logs': logs,
                        'testlog': None,
                        'languages': languages
                        }

                    # Fix up changes in path
                    if os.path.exists(os.path.join(root + logs, 'gcc-test.txt')):
                        builds[name][host]['testlog'] = logs + '/gcc-test.txt'

                    if os.path.exists(os.path.join(root + logs, 'gcc-testsuite.txt')):
                        builds[name][host]['testlog'] = logs + '/gcc-testsuite.txt'

    return hosts, cpus, builds

class buildlog:
    def GET(self):
        hosts, cpus, builds = find_builds()

        helpers = {}
        helpers['sorted'] = sorted

        return render.buildlog(hosts, cpus, builds, helpers)

class tickets:
    def GET(self):
        helpers = {}
        helpers['sorted'] = sorted
        helpers['getattr'] = getattr

        return render.tickets(load_db(), helpers, web.input().get('group_by', None))

class login:
    def GET(self):
        return render.login(web.webopenid.status())

class patchtrackcreate:
    createform = web.form.Form(
        web.form.Textbox('summary', web.form.notnull, description='Summary', size=80),
        web.form.Textbox('project', web.form.notnull, description='Project', size=80),
        web.form.Textbox('series', web.form.notnull, description='Series', size=80),
        web.form.Textbox('assignee', web.form.notnull, description='Assignee', size=80),
        web.form.Textbox('milestone', web.form.notnull, description='Upstream milestone', size=80),
        web.form.Textbox('tags', description='Tags', size=80),
        web.form.Textarea('comment', web.form.notnull, description='Comment', rows=20, cols=80)
        )

    def validate(self, path):
        user = authenticate()

        match = re.match(r'(lp:\S+),revno=(\d+)', path)

        if not match:
            raise web.seeother('/helpers/patchtrack?msg=Invalid argument')

        branch, revno = match.groups()

        db = load_db()
        rev = db['revision_map'].get(path, None)

        if not rev:
            raise web.seeother('/helpers/patchtrack?msg=Unrecognised revision')

        branches = {
            'lp:gcc-linaro/4.5': { 'project': 'gcc-linaro', 'series': '4.5', 'milestone': '4.6.0' },
            'lp:gcc-linaro/4.4': { 'project': 'gcc-linaro', 'series': '4.4', 'milestone': '4.6.0' },
            None: { 'project': '<project>', 'series': '<series>' },
            }

        details = branches.get(branch, branches[None])

        return rev, details, user, branches.get(branch, None)

    def GET(self, path):
        rev, details, user, branch = self.validate(path)

        form = self.createform()
        form.summary.set_value('[%s:r%s] %s' % (details['series'], rev.revno, rev.short))
        form.project.set_value(details['project'])
        form.series.set_value(details['series'])
        form.assignee.set_value(rev.mapped_authors[0].name if rev.mapped_authors else user)
        form.milestone.set_value(details['milestone'])
        form.tags.set_value('revision')

        comment = []

        if rev.fixes:
            for fixed in rev.fixes:
                comment.append('Fixes: %s' % fixed)

        comment.append('Related: %s' % path)
        comment.append('')
        comment.append(rev.message)

        form.comment.set_value('\n'.join(comment))

        return render.patchtrackcreate(user, path, rev, branch, form, [])

    def POST(self, path):
        rev, details, user, branch = self.validate(path)

        form = self.createform()
        input = web.input()

        if form.validates():
            errors = []

            # Create the ticket
            lp = create_lp()

            # Validate all of the inputs
            try:
                # Project...
                project = lp.projects[input.project]
                try:
                    # Tracking project...
                    tracking = lp.projects[input.project + '-tracking']

                    try:
                        # Milestone on tracking project...
                        upstream = tracking.getMilestone(name=input.milestone)
                    except:
                        upstream = None

                    if not upstream:
                        errors.append('No such milestone %s on %s-tracking' % (input.milestone, input.project))

                except:
                    errors.append('No such tracking project %s-tracking' % input.project)

                try:
                    # Series on project...
                    series = project.getSeries(name=input.series)
                except:
                    series = None

                if not series:
                    errors.append('No such series %s on %s' % (input.series, input.project))
            except:
                errors.append('No such project %s' % input.project)

            try:
                # Assignee...
                assignee = lp.people[input.assignee]
            except:
                errors.append('No such assignee %s' % input.assignee)

            if errors:
                return render.patchtrackcreate(user, path, rev, branch, form, errors)
            else:
                try:
                    # # Create the bug
                    # bug = lp.bugs.createBug(title=input.summary, description=input.comment, tags=input.tags, target=project)

                    # # Hook it into the series
                    # task = bug.addTask(target=series)
                    # task.assignee = assignee
                    # task.lp_save()

                    # # Hook it into the tracking project
                    # task = bug.addTask(target=tracking)
                    # task.milestone = upstream
                    # task.assignee = assignee
                    # task.lp_save()

                    # # Done
                    raise web.seeother('/helpers/patchtrack?msg=Ticket created')
                except Exception, ex:
                    errors.append('Exception while talking with Launchpad: %s' % ex)
                    return render.patchtrackcreate(user, path, rev, branch, form, errors)
        else:
            return render.patchtrackcreate(user, path, rev, branch, form, [])

class testcompare:
    def GET(self, name):
        bases = [
            ('gcc-linaro-4.5', 'gcc-4.5.1-0'),
            ('gcc-linaro-4.4', 'gcc-4.4.4-0'),
            ]

        for match, ref in bases:
            if name.startswith(match):
                base = ref
                break
        else:
            base = None

        # Find the corresponding build
        parts = name.split('/')[2].split('-')
        logs = map_filename('%s/logs/%s-*/gcc-test*.txt' % (base, parts[0]))

        baselog = glob.glob(logs)[-1]

        a = gcchelpers.parse(map_filename(baselog))
        b = gcchelpers.parse(map_filename(name))

        helpers = {}
        helpers['sorted'] = sorted
        helpers['getattr'] = getattr

        return render.testcompare(a, b, helpers)

if __name__ == "__main__":
    if running_on('crucis'):
        # Development machine.  Run stand alone
        pass
    else:
        web.wsgi.runwsgi = lambda func, addr=None: web.wsgi.runfcgi(func, addr)

    app.run()