~ubuntu-security/ubuntu-security-tools/trunk

#!/usr/bin/python

# Copyright (C) 2007-2014 Canonical, Ltd.

# Author: Kees Cook <kees@ubuntu.com>

#         Jamie Strandboge <jamie@canonical.com>

#         Marc Deslauriers <marc.deslauriers@canonical.com>

# License: GPLv3

import sys, subprocess, tempfile, re, os

import optparse

def pull_build_log(filename):

    sep = '\nChecking correctness of source dependencies...\n'

    log = file(filename).read()

    if sep in log:

        log = log.split(sep,1)[1]

#        log = sep.join(log.split(sep)[:-1])+"\n"

    return log

def sort_build(log, pkg=None):

    out = []

    pat = re.compile(r'^\s+')

    pat_beg = None # used for matching at the beginning of a line to ensure

                   # the next line gets added to it

    pat_force_line = None # used to match at beginning of line to ensure it

                          # is on its own line

    if pkg.startswith("python2") or pkg.startswith("python3"):

        pat = re.compile(r"^(\s+|Expecting.+|ok$|FAIL|ERROR)")

        pat_beg = re.compile(r'^Trying:')

    elif pkg.startswith("openjdk"):

        pat_force_line = re.compile(r"^\s+(created|inflated|inflating): ")

    pat_w = re.compile(r'[a-zA-Z0-9]')

    s = ""

    for line in log.split('\n'):

        if pat_beg and pat_beg.search(line):

            s = line

        elif pat.search(line) and \

             not (pat_force_line and pat_force_line.search(line)):

            if s == "": # avoids adding extra newline

                s = line

            else:

                s += 'BLSNEWLINE' + line

        else:

            if s == "": # avoids adding extra newline

                s = line

            else:

                s = "%s\n%s" % (s, line)

            if pat_w.search(s):

                # Don't include anything that isn't worth sorting

                out.append(s)

            s = ""

    out.sort()

    out_s = "\n".join(out)

    # Now that we are sorted, replace BLSNEWLINE placeholders

    pat = re.compile(r'BLSNEWLINE')

    return pat.sub('\n', out_s)

def write_log(fd, log, deparallel, pkg):

    if deparallel:

        split_str = '\n debian/rules build\n'

        (pre, rest) = log.split(split_str, 1)

        pre += split_str

        split_str = 'fakeroot debian/rules binary-arch\n'

        if split_str not in rest:

            split_str = 'fakeroot debian/rules binary\n'

        (build, post) = rest.split(split_str, 1)

        post = split_str + post

        # Don't try to sort non-build targets

        fd.write(pre)

        # Sort the build target

        fd.write(sort_build(build, pkg))

        # Don't try to sort non-build targets

        fd.write(post)

        fd.write("Specified --deparallel with '%s' (experimental)\n" % fd.name)

    else:

        fd.write(log)

    fd.flush()

parser = optparse.OptionParser()

parser.add_option("--deparallel", help="Attempt to deparallelize the build logs", action='store_true', default=False)

(opt, args) = parser.parse_args()

if len(args)<4:

    sys.stderr.write("Usage: %s [--deparallel] PKG OLDVER OLDLOG NEWVER NEWLOG\n" % (os.path.basename(sys.argv[0])))

    sys.exit(1)

pkg = args[0]

oldver = args[1]

oldver_escaped = oldver.replace('.','\.')

oldlog = pull_build_log(args[2]).replace(oldver,"VERSION").replace(oldver_escaped,"VERSION")

if '-' in oldver:

    oldverup = pkg + "-" + '-'.join(oldver.split('-')[:-1])

    oldlog = oldlog.replace(oldverup,"%s-VERSION"%(pkg))

newver = args[3]

newver_escaped = newver.replace('.','\.')

newlog = pull_build_log(args[4]).replace(newver,"VERSION").replace(newver_escaped,"VERSION")

if '-' in newver:

    newverup = pkg + "-" + '-'.join(newver.split('-')[:-1])

    newlog = newlog.replace(newverup,"%s-VERSION"%(pkg))

buildpath_regex = r'\/build\/([^\/]+)\/%s-VERSION[\/\'\" ]' % (pkg)

# Normalize Ubuntu buildd build host (must happen before USER normalization)

oldlog = re.sub(r'<buildd@.*\.buildd>', '<buildd@BUILDHOST.buildd>', oldlog)

newlog = re.sub(r'<buildd@.*\.buildd>', '<buildd@BUILDHOST.buildd>', newlog)

# Normalize user

userold = "USER"

if re.search(buildpath_regex, oldlog) != None:

    try:

        userold = re.search(buildpath_regex, oldlog).group(1)

    except:

        sys.stderr.write("Problem processing '%s' (perhaps an aborted build?)\n" % (args[2]))

        sys.exit(1)

usernew = "USER"

if re.search(buildpath_regex, newlog) != None:

    try:

        usernew = re.search(buildpath_regex, newlog).group(1)

    except:

        sys.stderr.write("Problem processing '%s' (perhaps an aborted build?)\n" % (args[4]))

        sys.exit(1)

if '-' in userold:

    userold = userold.split('-')[0]

if '-' in usernew:

    usernew = usernew.split('-')[0]

oldlog = re.sub(r'\b%s\b' % (userold), 'USER', oldlog)

newlog = re.sub(r'\b%s\b' % (usernew), 'USER', newlog)

# Normalize build paths

#/build/buildd/                                                 gdb-6.8.90.20090918/...

#/build/kees-gdb_6.8.90.20090918-0ubuntu1~ppa2-amd64-WX65oZ     gdb-6.8.90.20090918/...

#-make[1]: Leaving directory `/build/buildd/gdb-VERSION'

#+make[1]: Leaving directory `/build/kees-gdb_VERSION-amd64-WX65oZ/gdb-VERSION'

#- ... -I"/build/jamie-libvpx_VERSION-amd64-xVTtDS/libvpx-VERSION" ...

#+ ... -I"/build/jamie-libvpx_VERSION-amd64-O1LsQi/libvpx-VERSION" ...

oldlog = re.sub(buildpath_regex, '/BUILDPATH/', oldlog)

newlog = re.sub(buildpath_regex, '/BUILDPATH/', newlog)

oldlog = re.sub(r'\/home\/USER\/build-[-0-9]+\/', '/HOMEBUILDPATH/', oldlog)

newlog = re.sub(r'\/home\/USER\/build-[-0-9]+\/', '/HOMEBUILDPATH/', newlog)

# Normalize Ubuntu buildd SUDO_COMMAND

oldlog = re.sub(r'\/home\/buildd\/build-[a-f0-9]+\/chroot-autobuild', '/BUILDPATH/chroot-autobuild', oldlog)

newlog = re.sub(r'\/home\/buildd\/build-[a-f0-9]+\/chroot-autobuild', '/BUILDPATH/chroot-autobuild', newlog)

# Normalize sbuild resolver paths

oldlog = re.sub(r'\/resolver-[a-zA-Z0-9_]+\/(apt_archive\/sbuild-build-depends-)', '/resolver-RESOLVERPATH/\\1', oldlog)

newlog = re.sub(r'\/resolver-[a-zA-Z0-9_]+\/(apt_archive\/sbuild-build-depends-)', '/resolver-RESOLVERPATH/\\1', newlog)

# Normalize umt tmp paths

oldlog = re.sub(r'\/tmp\/umt-[a-zA-Z0-9_]+', '/tmp/umt-TMPPATH', oldlog)

newlog = re.sub(r'\/tmp\/umt-[a-zA-Z0-9_]+', '/tmp/umt-TMPPATH', newlog)

# Normalize schroot mount points

oldlog = re.sub(r'/var/lib/schroot/mount/([^/]+)/', '/SCHROOTMOUNT/', oldlog)

newlog = re.sub(r'/var/lib/schroot/mount/([^/]+)/', '/SCHROOTMOUNT/', newlog)

# Normalize SCHROOT_SESSION_ID

oldlog = re.sub(r'SCHROOT_SESSION_ID=([a-z]+-[a-z0-9]+)-[0-9a-f\-]+', 'SCHROOT_SESSION_ID=\\1-SCHROOTSESSIONID', oldlog)

newlog = re.sub(r'SCHROOT_SESSION_ID=([a-z]+-[a-z0-9]+)-[0-9a-f\-]+', 'SCHROOT_SESSION_ID=\\1-SCHROOTSESSIONID', newlog)

# Normalize schroot NOTICEs

oldlog = re.sub(r"(I: NOTICE: Log filtering will replace ').*(' with ').*('.*)", '\\1VAR1\\2VAR2\\3', oldlog)

newlog = re.sub(r"(I: NOTICE: Log filtering will replace ').*(' with ').*('.*)", '\\1VAR1\\2VAR2\\3', newlog)

# Normalize build finished

oldlog = re.sub(r'((Build finished at|Finished at)) [0-9\-]+', '\\1 FINISHTIME', oldlog)

newlog = re.sub(r'((Build finished at|Finished at)) [0-9\-]+', '\\1 FINISHTIME', newlog)

# Normalize redundant summary line

oldlog = re.sub(r'(Build needed) [0-9]{2}:[0-9]{2}:[0-9]{2}, [0-9]+[kmg] (disc space)', '\\1 BUILDTIME, BUILDSPACE \\2', oldlog)

newlog = re.sub(r'(Build needed) [0-9]{2}:[0-9]{2}:[0-9]{2}, [0-9]+[kmg] (disc space)', '\\1 BUILDTIME, BUILDSPACE \\2', newlog)

# Normalize file times

oldlog = re.sub('(\d+ )[12][09]\d\d-\d\d-\d\d [0-2]\d:[0-5]\d(?::[0-5]\d)?( \./)', '\g<1>0000-00-00 00:00\g<2>', oldlog)

newlog = re.sub('(\d+ )[12][09]\d\d-\d\d-\d\d [0-2]\d:[0-5]\d(?::[0-5]\d)?( \./)', '\g<1>0000-00-00 00:00\g<2>', newlog)

# Normalize timestamps (e.g. Mon Aug 24 16:58:26 UTC 2009)

oldlog = re.sub('(?:(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun),? )?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [ \d]?\d [0-2]\d:[0-5]\d(?::[0-5]\d)?(?:(?: [^ ]+)? [12][09]\d\d|[-+]\d+)?', 'DAY MON DD HH:MM:SS TZ YEAR', oldlog)

newlog = re.sub('(?:(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun),? )?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [ \d]?\d [0-2]\d:[0-5]\d(?::[0-5]\d)?(?:(?: [^ ]+)? [12][09]\d\d|[-+]\d+)?', 'DAY MON DD HH:MM:SS TZ YEAR', newlog)

# Normalize ' size 2121828 bytes: control archive= 607 bytes.'

oldlog = re.sub(' size \d+ bytes: control archive= \d+ bytes', ' size NN bytes: control archive= NN bytes', oldlog)

newlog = re.sub(' size \d+ bytes: control archive= \d+ bytes', ' size NN bytes: control archive= NN bytes', newlog)

# progress dots (less than 4 catches too much)

oldlog = re.sub('\.\.\.\.\.*', '.4+DOTS.', oldlog)

newlog = re.sub('\.\.\.\.\.*', '.4+DOTS.', newlog)

# Normalize 'Ubuntu [a-z]+ \(development branch\)' and 'Ubuntu [\d]?\d\.(04|10)'

oldlog = re.sub('Ubuntu ([a-z]+ \(development branch\)|[\d]?\d\.(04|10))', 'Ubuntu YY.MM', oldlog)

newlog = re.sub('Ubuntu ([a-z]+ \(development branch\)|[\d]?\d\.(04|10))', 'Ubuntu YY.MM', newlog)

# Normalize chatty apt output

# Filter out 'Reading package lists...' bits

oldlog = re.sub(r'(Reading package lists...) [0-9]{1,2}%.*(Reading package lists... Done)', '\\1 PROGRESS... \\2', oldlog)

newlog = re.sub(r'(Reading package lists...) [0-9]{1,2}%.*(Reading package lists... Done)', '\\1 PROGRESS... \\2', newlog)

# Normalize 'Fetched ... in ...

oldlog = re.sub(r'Fetched [0-9\.]+ ?[kmMgGB]+ in( [0-9]+min)? [0-9]+s \([0-9\.]+ ?[kmMgGB]+/s\)', 'Fetched SIZE in TIME (RATE)', oldlog)

newlog = re.sub(r'Fetched [0-9\.]+ ?[kmMgGB]+ in( [0-9]+min)? [0-9]+s \([0-9\.]+ ?[kmMgGB]+/s\)', 'Fetched SIZE in TIME (RATE)', newlog)

# Normalize 'Need to get ...'

oldlog = re.sub(r'Need to get [0-9\.]+ ?[kmMgGB]+(/[0-9\.]+ ?[kmMgGB]+)? of archives', 'Need to get SIZE of archives', oldlog)

newlog = re.sub(r'Need to get [0-9\.]+ ?[kmMgGB]+(/[0-9\.]+ ?[kmMgGB]+)? of archives', 'Need to get SIZE of archives', newlog)

# Normalize 'Get:... vs Hit... for binaries'

oldlog = re.sub(r'(Get:[0-9]+|Hit) (\S+://\S+ \S+ \S+ \S+ \S+) \[[0-9\.]+ ?[kmMgGB]+\]', 'GETHITIGN \\2 SIZEOMITTED', oldlog)

newlog = re.sub(r'(Get:[0-9]+|Hit) (\S+://\S+ \S+ \S+ \S+ \S+) \[[0-9\.]+ ?[kmMgGB]+\]', 'GETHITIGN \\2 SIZEOMITTED', newlog)

# Normalize 'Get:... vs Hit... vs Ign... for Packages'

oldlog = re.sub(r'(Get:[0-9]+|Hit|Ign) (\S+://\S+ \S+ \S+ Packages\S*)( \[[0-9\.]+ ?[kmMgGB]+\])?', 'GETHITIGN \\2 SIZEOMITTED', oldlog)

newlog = re.sub(r'(Get:[0-9]+|Hit|Ign) (\S+://\S+ \S+ \S+ Packages\S*)( \[[0-9\.]+ ?[kmMgGB]+\])?', 'GETHITIGN \\2 SIZEOMITTED', newlog)

# Normalize 'Get:... vs Hit... vs Ign... for Sources, Release, etc'

oldlog = re.sub(r'(Get:[0-9]+|Hit|Ign) (\S+://\S+ \S+ \S+)( \[[0-9\.]+ ?[kmMgGB]+\])?', 'GETHITIGN \\2 SIZEOMITTED', oldlog)

newlog = re.sub(r'(Get:[0-9]+|Hit|Ign) (\S+://\S+ \S+ \S+)( \[[0-9\.]+ ?[kmMgGB]+\])?', 'GETHITIGN \\2 SIZEOMITTED', newlog)

# Normalize 'After this operation, ...

oldlog = re.sub(r'After this operation, [0-9]+ ?[kmMgGB]+ of additional disk space will be used', 'After this operation, SIZE of additional disk space will be used', oldlog)

newlog = re.sub(r'After this operation, [0-9]+ ?[kmMgGB]+ of additional disk space will be used', 'After this operation, SIZE of additional disk space will be used', newlog)

# /tmp files

oldlog = re.sub('/tmp/.*', '/tmp/TMPFILE', oldlog)

newlog = re.sub('/tmp/.*', '/tmp/TMPFILE', newlog)

# dpkg-gensymbols

oldlog = re.sub('dpkg-gensymbols[a-zA-Z0-9_]+\s+\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ [+-]\d{4}', 'dpkg-gensymbolsXXXX      DATETIME', oldlog)

newlog = re.sub('dpkg-gensymbols[a-zA-Z0-9_]+\s+\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ [+-]\d{4}', 'dpkg-gensymbolsXXXX      DATETIME', newlog)

# Normalize special stuff

# openjdk...

oldlog = re.sub('Timing: \d+ seconds or (?:\d+m)?\d+s for', 'Timing: NNNNN seconds or NmNs for', oldlog)

newlog = re.sub('Timing: \d+ seconds or (?:\d+m)?\d+s for', 'Timing: NNNNN seconds or NmNs for', newlog)

oldlog = re.sub('-D(FULL_VERSION|RELEASE)=\'"([a-z0-9_\.\-]+)"\'', '-D\\1=\'"N.N.N_NN"\'', oldlog)

newlog = re.sub('-D(FULL_VERSION|RELEASE)=\'"([a-z0-9_\.\-]+)"\'', '-D\\1=\'"N.N.N_NN"\'', newlog)

oldlog = re.sub('(BUILD_NUMBER|JDK_BUILD_NUMBER|JDK_MKTG_VERSION|JDK_VERSION|COOKED_JDK_UPDATE_VERSION|COOKED_BUILD_NUMBER|JRE_RELEASE_VERSION|HOTSPOT_RELEASE_VERSION|FULL_VERSION)=(\\")?([a-zA-Z0-9_\.\-]+)(\\")?', '\\1=N.N.N_NN', oldlog)

newlog = re.sub('(BUILD_NUMBER|JDK_BUILD_NUMBER|JDK_MKTG_VERSION|JDK_VERSION|COOKED_JDK_UPDATE_VERSION|COOKED_BUILD_NUMBER|JRE_RELEASE_VERSION|HOTSPOT_RELEASE_VERSION|FULL_VERSION)=(\\")?([a-z0-9_\.\-]+)(\\")?', '\\1=N.N.N_NN', newlog)

oldlog = re.sub('(lib.*.so(\.[0-9]+)? => .*lib.*.so(\.[0-9])?) \(0x[0-9a-f]+\)', '\\1 (0xDEADBEEF)', oldlog) 

newlog = re.sub('(lib.*.so(\.[0-9]+)? => .*lib.*.so(\.[0-9])?) \(0x[0-9a-f]+\)', '\\1 (0xDEADBEEF)', newlog) 

oldlog = re.sub('#@@RELEASE@@#[a-z0-9_\.\-]+#', '#@@RELEASE@@#N.N.N_NN#', oldlog)

newlog = re.sub('#@@RELEASE@@#[a-z0-9_\.\-]+#', '#@@RELEASE@@#N.N.N_NN#', newlog)

oldlog = re.sub('DERIVATIVE_ID="(\\")?IcedTea[0-9]+ ([0-9\.]+)(\\")?"', 'DERIVATIVE_ID="\\"IcedTeaN N.N.N\\""', oldlog)

newlog = re.sub('DERIVATIVE_ID="(\\")?IcedTea[0-9]+ ([0-9\.]+)(\\")?"', 'DERIVATIVE_ID="\\"IcedTeaN N.N.N\\""', newlog)

oldlog = re.sub('-DDERIVATIVE_ID="(\\")?IcedTea[0-9]+ ([0-9\.]+)(\\")?"', '-DDERIVATIVE_ID="\\"IcedTeaN N.N.N\\""', oldlog)

newlog = re.sub('-DDERIVATIVE_ID="(\\")?IcedTea[0-9]+ ([0-9\.]+)(\\")?"', '-DDERIVATIVE_ID="\\"IcedTeaN N.N.N\\""', newlog)

# gtk+3.0

oldlog = re.sub('/build/gtk\+3.0-[^/]+/', '/build/gtk+3.0-TEMP/', oldlog)

newlog = re.sub('/build/gtk\+3.0-[^/]+/', '/build/gtk+3.0-TEMP/', newlog)

# openssh

oldlog = re.sub('SSH_EXTRAVERSION=\\\\\"[a-zA-Z0-9_\.\-]+\\\\\"', 'SSH_EXTRAVERSION=\\\\"XXXX\\\\\"', oldlog)

newlog = re.sub('SSH_EXTRAVERSION=\\\\\"[a-zA-Z0-9_\.\-]+\\\\\"', 'SSH_EXTRAVERSION=\\\\"XXXX\\\\\"', newlog)

oldlog = re.sub('SSH_EXTRAVERSION=\\\"[a-zA-Z0-9_\.\-]+\\\"', 'SSH_EXTRAVERSION=\\"XXXX\\\"', oldlog)

newlog = re.sub('SSH_EXTRAVERSION=\\\"[a-zA-Z0-9_\.\-]+\\\"', 'SSH_EXTRAVERSION=\\"XXXX\\\"', newlog)

# libreoffice (adding: com/sun/... (deflated 60%))

oldlog = re.sub(' \(deflated [0-9]{1,2}%\)', ' (deflated N%)', oldlog)

newlog = re.sub(' \(deflated [0-9]{1,2}%\)', ' (deflated N%)', newlog)

# samba (09:33:21 runner /usr/bin/gcc -g -O2 -fstack-protector)

oldlog = re.sub('[0-9:]{8} runner', '00:00:00 runner', oldlog)

newlog = re.sub('[0-9:]{8} runner', '00:00:00 runner', newlog)

# PNG optimization

oldlog = re.sub('(pkgstripfiles: PNG optimization for package \S+ took) .*', '\\1 TIME', oldlog)

newlog = re.sub('(pkgstripfiles: PNG optimization for package \S+ took) .*', '\\1 TIME', newlog)

# gi introspection (as seen in rhythmbox)

oldlog = re.sub('/tmp-introspect[a-zA-Z0-9_]+/', '/tmp-XXXX/', oldlog)

newlog = re.sub('/tmp-introspect[a-zA-Z0-9_]+/', '/tmp-XXXX/', newlog)

# python (flags=re.MULTILINE is 2.7 only, otherwise use re.compile(... flags=))

oldlog = re.sub(r'^Ran ([0-9]+) tests in [0-9]+\.[0-9]+s$', 'Ran \\1 tests in N.NNNs', oldlog, flags=re.MULTILINE)

newlog = re.sub(r'^Ran ([0-9]+) tests in [0-9]+\.[0-9]+s$', 'Ran \\1 tests in N.NNNs', newlog, flags=re.MULTILINE)

oldlog = re.sub(r'(<.* object at) 0x[0-9a-f]+>$', '\\1 0xHHHHHHHH>', oldlog, flags=re.MULTILINE)

newlog = re.sub(r'(<.* object at) 0x[0-9a-f]+>$', '\\1 0xHHHHHHHH>', newlog, flags=re.MULTILINE)

oldlog = re.sub(r"^( (|a )server:  new connection from \('127.0.0.1',) [0-9]+\)$", '\\1 PORT)', oldlog, flags=re.MULTILINE)

newlog = re.sub(r"^( (|a )server:  new connection from \('127.0.0.1',) [0-9]+\)$", '\\1 PORT)', newlog, flags=re.MULTILINE)

oldlog = re.sub(r"( server:  new connection from 127.0.0.1:)[0-9]+$", '\\1PORT)', oldlog, flags=re.MULTILINE)

newlog = re.sub(r"( server:  new connection from 127.0.0.1:)[0-9]+$", '\\1PORT)', newlog, flags=re.MULTILINE)

# gcc AddressSanitzer Failures; remove pid

oldlog = re.sub('(FAIL: .*, is )====(ERROR: AddressSanitizer failed to allocate 0x)[0-9a-f]+ \(\d+\) (bytes at address 0x)[0-9a-f]+ (.*)', '\\1 ==PID==\\2HEXSIZEOMITTED (SIZEOMITTED) \\3ADDROMITTED \\4)', oldlog)

newlog = re.sub('(FAIL: .*, is )==\d+==(ERROR: AddressSanitizer failed to allocate 0x)[0-9a-f]+ \(\d+\) (bytes at address 0x)[0-9a-f]+ (.*)', '\\1 ==PID==\\2HEXSIZEOMITTED (SIZEOMITTED) \\3ADDROMITTED \\4)', newlog)

# Normalize tomcat7 junit timestamps (e.g. Jul 24, 2014 2:02:12 PM)

oldlog = re.sub('\[junit\] (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [ \d]?\d, [12][09]\d\d 1?\d:[0-5]\d:[0-5]\d (?:AM|PM)', 'MON DD, YEAR HH:MM:SS XM', oldlog)

newlog = re.sub('\[junit\] (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [ \d]?\d, [12][09]\d\d 1?\d:[0-5]\d:[0-5]\d (?:AM|PM)', 'MON DD, YEAR HH:MM:SS XM', newlog)

# Normalize tomcat7 pids

oldlog = re.sub('http-bio-127.0.0.1-auto-[0-9]+-[0-9]+', 'http-bio-127.0.0.1-auto-X-XXX', oldlog)

newlog = re.sub('http-bio-127.0.0.1-auto-[0-9]+-[0-9]+', 'http-bio-127.0.0.1-auto-X-XXX', newlog)

# Normalize tomcat7 time elapsed

oldlog = re.sub('Time elapsed: [0-9]+.[0-9]+ sec', 'Time elapsed: X.XXX sec', oldlog)

newlog = re.sub('Time elapsed: [0-9]+.[0-9]+ sec', 'Time elapsed: X.XXX sec', newlog)

# Normalize icu test suite time

oldlog = re.sub(' \(?\( ?[0-9.]+m?s ?\)\)?', '', oldlog)

newlog = re.sub(' \(?\( ?[0-9.]+m?s ?\)\)?', '', newlog)

oldlog = re.sub('Elapsed Time: [0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}', 'Elapsed Time: XX:XX:XX.XXX', oldlog)

newlog = re.sub('Elapsed Time: [0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}', 'Elapsed Time: XX:XX:XX.XXX', newlog)

# Normalize openldap test suite

oldlog = re.sub('PID=\d+([ :])', 'PID=PID\\1', oldlog)

newlog = re.sub('PID=\d+([ :])', 'PID=PID\\1', newlog)

oldlog = re.sub('\(pid=\d+\)', '(pid=PID)', oldlog)

newlog = re.sub('\(pid=\d+\)', '(pid=PID)', newlog)

oldlog = re.sub('in [0-9]+.[0-9]+ seconds', 'in X.XXX seconds', oldlog)

newlog = re.sub('in [0-9]+.[0-9]+ seconds', 'in X.XXX seconds', newlog)

# Normalize openssl test suite

oldlog = re.sub('[0-9]{15}:error', '000000000000000:error', oldlog)

newlog = re.sub('[0-9]{15}:error', '000000000000000:error', newlog)

oldlog = re.sub('s3_clnt.c:[0-9]{4}', 's3_clnt.c:0000', oldlog)

newlog = re.sub('s3_clnt.c:[0-9]{4}', 's3_clnt.c:0000', newlog)

oldlog = re.sub('rsa_sign.c:[0-9]{3}', 'rsa_sign.c:000', oldlog)

newlog = re.sub('rsa_sign.c:[0-9]{3}', 'rsa_sign.c:000', newlog)

oldlog = re.sub('context\? 0x0x[0-9a-f]+ a cert\? 0x0x[0-9a-f]+', 'context\? 0x0xHHHHHH a cert\? 0x0xHHHHHH', oldlog)

newlog = re.sub('context\? 0x0x[0-9a-f]+ a cert\? 0x0x[0-9a-f]+', 'context\? 0x0xHHHHHH a cert\? 0x0xHHHHHH', newlog)

oldlog = re.sub('Nonce: 0x[0-9A-F]{16}', 'Nonce: 0xHHHHHHHHHHHHHHHH', oldlog)

newlog = re.sub('Nonce: 0x[0-9A-F]{16}', 'Nonce: 0xHHHHHHHHHHHHHHHH', newlog)

# perl build directory

oldlog = re.sub('/build/perl\\\-.*/', '/build/perl\-XXXXXX/', oldlog)

newlog = re.sub('/build/perl\\\-.*/', '/build/perl\-XXXXXX/', newlog)

# samba progress

oldlog = re.sub('\[[ 0-9]{4}/[ 0-9]{4}\] Compiling', '[XXXX/XXXX] Compiling', oldlog)

newlog = re.sub('\[[ 0-9]{4}/[ 0-9]{4}\] Compiling', '[XXXX/XXXX] Compiling', newlog)

oldlog = re.sub('\[[ 0-9]{4}/[ 0-9]{4}\] Generating', '[XXXX/XXXX] Generating', oldlog)

newlog = re.sub('\[[ 0-9]{4}/[ 0-9]{4}\] Generating', '[XXXX/XXXX] Generating', newlog)

oldlog = re.sub('\[[ 0-9]{4}/[ 0-9]{4}\] Linking', '[XXXX/XXXX] Linking', oldlog)

newlog = re.sub('\[[ 0-9]{4}/[ 0-9]{4}\] Linking', '[XXXX/XXXX] Linking', newlog)

# Write out the files for diffing

oldfile = tempfile.NamedTemporaryFile(prefix='buildlog-')

write_log(oldfile, oldlog, opt.deparallel, pkg)

newfile = tempfile.NamedTemporaryFile(prefix='buildlog-')

write_log(newfile, newlog, opt.deparallel, pkg)

# To debug, useful to do:

#shutil.copy(oldfile.name, "/tmp/old")

#shutil.copy(newfile.name, "/tmp/new")

subprocess.call(['diff', '-ub', oldfile.name, newfile.name])