~brian-murray/+junk/bug-agent

#!/usr/bin/python3
# Author: Brian Murray <brian@canonical.com>
# Copyright (C) 2010 Canonical, Ltd.
# License: GPLv3
#
# This is a script for modifying bug reports in an automated fashion

import gzip
import logging
import lpl_common
import re
import tempfile

try:
    from urllib.request import urlopen
except ImportError:
    from urllib import urlopen

from apport.crashdb import get_crashdb
from apt import apt_pkg
from httplib2 import FailedToDecompressContent
from datetime import datetime
from datetime import timedelta
from lazr.restfulclient import errors
from lazr.restfulclient.errors import ClientError, NotFound, ServerError


def bugs_from_changes(change_url):
    '''Return (bug_list, cve_list) from a .changes file URL'''
    changelog = urlopen(change_url)

    refs = []
    bugs = set()

    for l in changelog:
        if l.startswith('Launchpad-Bugs-Fixed: '):
            refs = l.split()[1:]
            break

    for b in refs:
        try:
            lpbug = lp.bugs[int(b)]
        except KeyError:
            continue
        bugs.add(lpbug)

    return sorted(bugs)

def log_print(msg):
    print(msg)
    logging.info(msg)

def tag_test_case(task):
    bug = task.bug
    if 'testcase' in bug.tags:
        return
    if 'TEST CASE' in bug.description:
        add_tags(bug, ['testcase'])
        print('LP: #%s tagged testcase' % bug.id)
        logging.info('LP: #%s tagged testcase' % bug.id)


def read_gz_file(attachment, task):
    try:
        remotefd = attachment.data.open()
    except FailedToDecompressContent:
        log_print('Failure with gzip read of %s from LP: #%s'
            % (attachment.title, task.bug.id))
        return
    tmpfile = tempfile.NamedTemporaryFile(dir='/tmp')
    datafile = open(tmpfile.name, 'w')
    datafile.write(remotefd.read())
    remotefd.close()
    datafile.close()

    gzipfd = gzip.open(tmpfile.name)
    try:
        contents = gzipfd.read()
    except IOError:
        log_print('Failure with gzip read of %s from LP: #%s'
            % (attachment.title, task.bug.id))
        if attachment.title == 'VarLogDistupgradeTermlog.gz':
            # workaround LP: #249950
            try:
                attachment.removeFromBug()
            except errors.HTTPError:
                pass
        return
    gzipfd.close()
    contents = contents.split('\n')
    return contents


def package_version_releases(package):
    primary = [arc for arc in ubuntu.archives if arc.name == 'primary'][0]
    versions = {}
    for ps in primary.getPublishedSources(source_name=package):
        versions[ps.source_package_version] = ps.distro_series.name
    return versions


def skip_bug(task):
    bug = task.bug
    if 'bot-stop-nagging' in bug.tags:
        return True
    return False


def only_reporter_comments(bug):
    commenters = []
    for message in bug.messages:
        try:
            commenters.append(message.owner.name)
        except ClientError as error:
            if error.response['status'] == '410':
                continue
    commenters = set(commenters)
    try:
        commenters.remove(bug.owner.name)
    except KeyError as error:
        print('%s: %s' % (bug.id, error))
    # the janitor adds empty comments so don't include him in the list of
    # commenters
    if 'janitor' in commenters:
        commenters.remove('janitor')
    if len(commenters) == 0:
        return True
    else:
        return False


def real_comment(task, skipped_commenters):
    '''Check for any comments from people not in skipped_commenters.'''

    for message in task.bug.messages:
        try:
            if message.owner.name not in skipped_commenters:
                return True
        except ClientError as error:
            if error.response['status'] == '410':
                continue
    return False


def trim_dpkg_log(log):
    '''Trim DpkgTerminalLog to the most recent installation session.'''

    lines = []
    trim_re = re.compile(b'^\(.* ... \d+ .*\)$')
    for line in log: #.splitlines():
        if trim_re.match(line):
            lines = []
            continue
        lines.append(str(line))
    return '\n'.join(lines)


def verification_nag(bug, pkg, release):
    '''Ask for verification on a bug'''

    V_NEEDED = ('The fix for this bug has been awaiting testing '
        'feedback in the -proposed repository for %s for more than 90 '
        'days.  Please test this fix and update the bug appropriately '
        'with the results.  In the event that the fix for this bug is '
        'still not verified 15 days from now, the package will be '
        'removed from the -proposed repository.' % release)
    bug_num = bug.id
    print("%s fixing LP: #%s has been in %s-proposed for more than 90 days"
        % (pkg, bug_num, release))
    # comment on the bug that it still needs verification
    bug.newMessage(content=V_NEEDED,
        subject='[%s/%s] verification still needed' % (pkg, release))
    # add a removal-candidate tag too
    tags = bug.tags
    tags.append('removal-candidate')
    bug.tags = tags
    bug.lp_save()
    print("LP: #%s asked for verification" % bug_num)
    logging.warning("LP: #%s asked for verification" % bug_num)

def error_in_attachments(task, error, attachment_names):
    '''Check a bug tasks's attachments for an error.'''

    # 2011-04-28 - this should be made more generic to work with more than
    # just dpkg attachments
    attachments = [a for a in task.bug.attachments if a.title in attachment_names]
    for attachment in attachments:
        # something more than just checking the file extension would be
        # better
        if attachment.title.endswith('txt'):
            hosted_file = attachment.data
            try:
                hosted_file_buffer = hosted_file.open()
            except OSError as error:
                if error.strerror == 'Connection refused':
                    continue
            except ServerError:
                continue
            # 2011-04-21 trim_dpkg_log shouldn't be used with just any
            # attachment
            trimmed_log = trim_dpkg_log(hosted_file_buffer)
            if isinstance(error, type('str')):
                if error in trimmed_log:
                    return True
            elif isinstance(error, type(re.compile('foo'))):
                if re.search(error, trimmed_log):
                    return True
        elif attachment.title.endswith('gz'):
            try:
                remotefd = attachment.data.open()
            except OSError as error:
                if error.strerror == 'Connection refused':
                    continue
            except ServerError:
                continue
            tmpfile = tempfile.NamedTemporaryFile(dir='/tmp')
            datafile = open(tmpfile.name, 'w')
            datafile.write(remotefd.read())
            remotefd.close()
            datafile.close()

            gzipfd = gzip.open(tmpfile.name)
            try:
                contents = gzipfd.read()
            except IOError:
                print('Failure with gzip read of %s from LP: #%s'
                    % (attachment.title, task.bug.id))
                logging.warning('Failure with gzip read of %s from LP: #%s'
                    % (attachment.title, task.bug.id))
                if attachment.title == 'VarLogDistupgradeTermlog.gz':
                    # workaround LP: #249950
                    try:
                        attachment.removeFromBug()
                    except errors.HTTPError:
                        pass
                continue
            if attachment.title == 'DpkgTerminalLog.gz':
                trimmed_log = trim_dpkg_log(contents)
            else:
                trimmed_log = contents
            if isinstance(error, type('str')):
                if error in trimmed_log:
                    return True
            elif isinstance(error, type(re.compile('foo'))):
                if re.search(error, trimmed_log):
                    return True


def mark_as_duplicate(master_number, bug, comment=None):
    '''Mark a bug report as a duplicate of another.'''

    master = lp.bugs[master_number]
    if comment is None:
        comment = 'Thank you for taking the time to report this bug and helping to make Ubuntu better. This particular bug has already been reported and is a duplicate of bug %s, so it is being marked as such. Please look at the other bug report to see if there is any missing information that you can provide, or to see if there is a workaround for the bug.  Additionally, any further discussion regarding the bug should occur in the other report.  Please continue to report any other bugs you may find.\n%s' % (master_number, AUTO_RESPONSE)
    if not comment == '':
        bug.newMessage(content=comment)
    bug.duplicate_of = master
    bug.lp_save()


def add_tags(bug, tags):
    '''Add a tag to a bug report.'''

    bug_num = bug.id
    bug_tags = bug.tags
    for tag in tags:
        if tag in bug_tags:
            # remove it so our print is accurate
            tags.remove(tag)
            continue
        bug_tags.append(tag)
        bug.tags = bug_tags
        bug.lp_save()
    if len(tags) == 0:
        return
    print('LP: #%s tagged %s' % (bug_num, ' '.join(tags)))
    logging.info('LP: #%s tagged %s' % (bug_num, ' '.join(tags)))


def full_live_media(task):
    '''Comment on and invalidate bug reports due to full /cdrom file systems on live media'''

    disk_full_comment = "Thanks for taking the time to report this bug and helping to make Ubuntu even better.  Examing the bug report it appears that you are using Ubuntu in a persistent mode on Live Media and that you have completely filled your /cdrom file system.  When using persistent mode your root (/) filesystem is a combination of a persistence file and the /cdrom file system which really contains /.  It is possible that you may be able to free some space using the command 'apt-cache clean' to remove packages from your local cache.  However, you may want to allocate more space to your real root partition."

    bug = task.bug
    if 'LiveMediaBuild' not in bug.description:
        return
    bug.newMessage(content=disk_full_comment)
    add_tags(bug, ['full-live-root'])
    task.status = 'Invalid'
    task.lp_save()
    bug_number = bug.id
    print('LP: #%s set to Invalid as it had a full root filesytem on live media'
        % bug_number)
    logging.info('LP: #%s set to Invalid as it had a full root filesytem on live media'
        % bug_number)


def reassign_initramfs_tools_bugs():
    '''LP: #580419 cleanup

    Move apport-package bug reports from the initramfs-tools package if they
    are about a different source package.
    '''
    initramfs = ubuntu.getSourcePackage(name='initramfs-tools')
    tasks = initramfs.searchTasks(tags=['apport-package'],
                                  order_by='-datecreated')
    for task in tasks[:50]:
        if skip_bug(task):
            continue
        if len([task for task in task.bug.bug_tasks]) != 1:
            continue
        if task.status != 'New':
            continue
        if only_reporter_comments(task.bug) is False:
            continue
        for line in task.bug.description.split('\n'):
            if line.startswith('DistroRelease:'):
                if '11.04' not in line:
                    break
            if line.startswith('Package:'):
                if 'initramfs-tools' in line or 'linux-image' in line:
                    break
                else:
                    package_name = line.split(' ')[1]
                    package = ubuntu.getSourcePackage(name=package_name)
                    task.target = package
                    task.lp_save()
                    print('Moving LP: #%s to %s' % (task.bug.id, package_name))
                    logging.info('LP: #%s package changed to %s' %
                        (task.bug.id, package_name))


def retag_linux_regression_updates():
    '''Retag r-update bugs r-release as dev release has no updates.'''

    linux = ubuntu.getSourcePackage(name='linux')
    bugs = [task.bug for task in linux.searchTasks(tags=['regression-update'],
            order_by='-datecreated')]
    for bug in bugs:
        tags = bug.tags
        if 'natty' in tags:
            tags.remove('regression-update')
            tags.append('regression-release')
            bug.tags = tags
            bug.lp_save()
            print('LP: #%s tagged regression-release' % bug.id)
            logging.info('LP: #%s tagged regression-release' % bug.id)


def retag_regression_potential():
    '''The r-potential tag has been deprecated and replaced by r-release.'''
    bugs = [task.bug for task in ubuntu.searchTasks(tags=['regression-potential'],
            order_by='-datecreated')]
    for bug in bugs[:10]:
        tags = bug.tags
        tags.remove('regression-potential')
        tags.append('regression-release')
        bug.tags = tags
        bug.lp_save()
        print('LP: #%s tagged regression-release' % bug.id)
        logging.info('LP: #%s tagged regression-release' % bug.id)


def tag_compiz_with_version():
    '''Tag compiz bugs with the major version number in use.'''

    compiz = ubuntu.getSourcePackage(name='compiz')
    bugs = [task.bug for task in compiz.searchTasks(tags=['apport-bug', 'apport-crash'],
            order_by='-datecreated')]
    for bug in bugs[:200]:
        tags = bug.tags
        if 'compiz-0.9' in tags or 'compiz-0.8' in tags or 'compiz-0.7' in tags:
            continue
        version = ''
        for line in bug.description.split('\n'):
            if line.startswith('Package:'):
                if 'compiz' in line.split(' ')[1]:
                    pkg_version = line.split(' ')[2]
                    version = pkg_version.replace('1:', '')[:3]
                    break
        if version != '' and version != '(no':
            tags.append('compiz-%s' % version)
            bug.tags = tags
            try:
                bug.lp_save()
            except:
                print('Failure in tag_compiz_with_version with LP: #%s'
                    % bug.id)
                logging.warning('Failure in tag_compiz_with_version with LP: #%s'
                    % bug.id)
            print('LP: #%s tagged compiz-%s' % (bug.id, version))
            logging.info('LP: #%s tagged compiz-%s' % (bug.id, version))

def open_upstream_project_task(package_name, project_name, since):
    '''For a package automatically open upstream project tasks.'''
    package = ubuntu.getSourcePackage(name=package_name)
    project = lp.projects[project_name]
    # hide_upstream is equivalent to "that are not known to affect upstream"
    ptasks = [task for task in package.searchTasks(tags=['-apport-package'],
              order_by='-datecreated', status_upstream='hide_upstream',
              created_since=since)]
    for ptask in ptasks:
        bug = ptask.bug
        btasks = [task for task in bug.bug_tasks]
        if len(btasks) != 1:
            btask_names = [task.bug_target_name for task in btasks]
            if project_name in btask_names:
                continue
        try:
            bug.addTask(target=project)
        except errors.HTTPError:
            log_print('LP: #%s skipped' % bug.id)
            continue
        print('LP: #%s added a %s project task' % (bug.id, project_name))
        logging.info('LP: #%s added a %s project task' % (bug.id,
            project_name))


def lzma_encoder_error(since):
    '''these are duplicates of LP: #691985 (lzma encoder error is vague)'''

    error = '-2147467259'
    comment = 'Thank you for taking the time to report this bug and helping to make Ubuntu better.  Unfortunately, the package installation error message that you have received is actually a failure with lzma and a generic one at that.  The following steps may help resolve the issue.  Please execute the following command, as it will clear your package cache, in a terminal:\n'
    comment += '\nsudo apt-get clean\n'
    comment += '\nThen try performing the update again as this may resolve your issue.  We plan to make lzma errors more informative and that work is being tracked in bug 691985 which I will mark this a duplicate of.'

    tasks = ubuntu.searchTasks(tags=['apport-package'],
                               order_by='-datecreated',
                               created_since=since)
    for task in tasks:
        if skip_bug(task):
            continue
        bug = task.bug
        bug_num = task.bug.id
        bug_shown = False

        attachments = [attachment for attachment in task.bug.attachments]
        for a in attachments:
            if bug_shown == True:
                break
            if 'DpkgTerminalLog.txt' in a.title:
                hosted_file = a.data
                hosted_file_buffer = hosted_file.open()
                trimmed_log = trim_dpkg_log(hosted_file_buffer)
                for line in trimmed_log.split('\n'):
                    if error in line:
                        bug_shown = True
                        # print('LP: #%s - %s' %
                        #       (bug_num, task.bug.title.encode('utf-8')))
                        add_tags(bug, ['lzma-efail'])
                        mark_as_duplicate('691985', bug, comment)
                        print('Marking LP: #%s as a duplicate of LP: #691985 (lzma encoder)' % bug_num)
                        logging.info('LP: #%s marked as a duplicate of LP: #691985 (lzma encoder)' % bug_num)
                        break
            elif 'DpkgTerminalLog.gz' in a.title or 'VarLogDistupgradeTermlog.gz' in a.title:
                remotefd = a.data.open()
                tmpfile = tempfile.NamedTemporaryFile(dir='/tmp')
                datafile = open(tmpfile.name, 'w')
                datafile.write(remotefd.read())
                remotefd.close()
                datafile.close()

                gzipfd = gzip.open(tmpfile.name)
                try:
                    contents = gzipfd.read()
                except IOError:
                    print('Failure with gzip read of %s from LP: #%s'
                        % (a.title, task.bug.id))
                    logging.info('Failure with gzip read of %s from LP: #%s'
                        % (a.title, task.bug.id))
                    if a.title == 'VarLogDistupgradeTermlog.gz':
                        # workaround LP: #249950
                        try:
                            a.removeFromBug()
                        except errors.HTTPError:
                            pass
                    continue
                for line in contents.split('\n'):
                    if error in line:
                        # print('LP: #%s - %s' %
                        #       (bug_num,task.bug.title.encode('utf-8')))
                        add_tags(bug, ['lzma-efail'])
                        mark_as_duplicate('691985', bug, comment)
                        print('Marking LP: #%s as a duplicate of LP: #691985 (lzma encoder)' % bug_num)
                        logging.info('LP: #%s marked as a duplicate of LP: #691985 (lzma encoder)' % bug_num)
                        break
                gzipfd.close()


def reassign_no_package_apport_bugs():
    '''Reassign apport-[crash|kerneloops|package] bugs to the right package'''

    tasks = ubuntu.searchTasks(status=['New'], has_no_package=True,
                               tags=['apport-crash', 'apport-kerneloops',
                                     'apport-package'],
                               tags_combinator='Any', order_by='-datecreated')

    for task in tasks:
        if skip_bug(task):
            continue
        skipped_commenters = ['apport']
        reporter = task.bug.owner.name
        skipped_commenters.append(reporter)
        tag_test_case(task)
        if 'needs-reassignment' in task.bug.tags:
            continue
        bug_num = task.bug.id
        # need to work on cranberry (lucid) too
        if len([task for task in task.bug.bug_tasks]) > 1:
            continue
        if task.bug.message_count > 2 and \
                real_comment(task, skipped_commenters) is True:
            print('LP: #%s (by %s) re no package was not reassigned.' %
                  (bug_num, reporter))
            logging.info('LP: #%s (by %s) re no package was not reassigned.' %
                         (bug_num, reporter))
            continue
        if task.bug_target_name == 'ubuntu':
            bug_task = task
            package_name = None
            for line in bug_task.bug.description.split('\n'):
                if 'SourcePackage: ' in line:
                    package_name = line.split(' ')[1]
            # per duflu Xorg isn't the best package either
            if package_name == 'xorg':
                continue
            if not package_name:
                print('LP: #%s has no SourcePackage line in the description' % bug_num)
                logging.info('LP: #%s has no SourcePackage line in the description' % bug_num)
                continue
            package = ubuntu.getSourcePackage(name=package_name)
            try:
                bug_task.target = package
                bug_task.lp_save()
                print('Moving bug LP: #%s to package %s' %
                    (bug_num, package.name))
                logging.info('LP: #%s moved to package %s' %
                    (bug_num, package.name))
            except errors.HTTPError as error:
                print('Failure in reassign_no_package_crashes with LP: #%s' %
                    bug_num)
                logging.warning('Failure in reassign_no_package_crashes with LP: #%s' %
                    bug_num)


def reassign_no_package_apportbug_bugs():
    '''Assign apport-bug bugs to the intended package'''

    blacklist = ['evince', 'gnome-terminal', 'nautilus', 'software-center',
        'yelp']

    tasks = ubuntu.searchTasks(status=['New', 'Confirmed', 'Triaged'],
                has_no_package=True, tags=['apport-bug'],
                               tags_combinator='Any', order_by='-datecreated')

    for task in tasks:
        if skip_bug(task):
            continue
        skipped_commenters = ['apport', 'janitor']
        try:
            skipped_commenters.append(task.bug.owner.name)
        except ClientError as error:
            if error.response['status'] == '410':
                pass
        tag_test_case(task)
        if 'needs-reassignment' in task.bug.tags:
            continue
        bug_num = task.bug.id
        # need to work on cranberry (lucid) too
        if len([task for task in task.bug.bug_tasks]) > 1:
            continue
        if task.bug.message_count > 2 and \
                real_comment(task, skipped_commenters) is True:
            continue
        target_set = [a.newvalue for a in task.bug.activity if a.whatchanged == 'affects']
        if 'ubuntu' in target_set:
            continue
        bug_task = task
        package_name = ''
        for line in bug_task.bug.description.split('\n'):
            if 'SourcePackage: ' in line:
                package_name = line.split(' ')[1]
                break
        if package_name in blacklist:
            continue
        # print('LP: #%s would be moved to %s' % (bug_num, package_name))
        package = ubuntu.getSourcePackage(name=package_name)
        try:
            bug_task.target = package
            bug_task.lp_save()
            print('Moving bug LP: #%s to package %s' %
                (bug_num, package.name))
            logging.info('LP: #%s moved to package %s' %
                (bug_num, package.name))
        except errors.HTTPError:
            print('Failure in reassign_no_package_bugs with LP: #%s' %
                bug_num)
            logging.warning('Failure in reassign_no_package_bugs with LP: #%s' %
                bug_num)


def no_package_generic_comment(since):
    '''Comment on no package bugs indicating the importance of having a
       package and where to get help.'''

    bdmurray = lp.people['brian-murray']

    apport_tags = set(['apport-bug', 'apport-crash', 'apport-package'])

    comment = "Thank you for taking the time to report this bug and helping to make Ubuntu better.  It seems that your bug report is not filed about a specific source package though, rather it is just filed against Ubuntu in general.  It is important that bug reports be filed about source packages so that people interested in the package can find the bugs about it.  You can find some hints about determining what package your bug might be about at https://wiki.ubuntu.com/Bugs/FindRightPackage.  You might also ask for help in the #ubuntu-bugs irc channel on Libera.chat.\n\nTo change the source package that this bug is filed about visit TASK_URL/+editstatus and add the package name in the text box next to the word Package.\n\n[This is an automated message.  I apologize if it reached you inappropriately; please just reply to this message indicating so.]"

    tasks = ubuntu.searchTasks(status=['New', 'Confirmed'],
                has_no_package=True, tags=['-needs-packaging', '-bot-comment'],
                tags_combinator='All', order_by='-datecreated',
                created_since=since)

    for task in tasks:
        if skip_bug(task):
            continue
        bug = task.bug
        if len([task for task in task.bug.bug_tasks]) > 1:
            continue
        low_title = bug.title.lower()
        # do not comment on feature freeze exceptions
        if '[ffe]' in low_title:
            continue
        # do not comment on sync requests
        if 'sync' in low_title:
            continue
        if 'needs-packaging' in low_title:
            add_tags(bug, ['needs-packaging'])
            continue
        tags = bug.tags
        if 'bot-comment' in tags:
            continue
        if 'needs-reassignment' in tags:
            continue
        if apport_tags.intersection(tags) and task.status == 'New':
            continue
        target_set = [a.newvalue for a in bug.activity if a.whatchanged == 'affects']
        if 'ubuntu' in target_set:
            continue
        if only_reporter_comments(bug) is False and task.status == 'New':
            continue
        modified_comment = comment.replace('TASK_URL', task.web_link)
        bug.newMessage(content=modified_comment)
        # 2011-11-21 too much email
        #bug.subscribe(person=bdmurray)
        bug_number = bug.id
        reporter = bug.owner.name
        add_tags(bug, ['bot-comment'])
        print('LP: #%s (by %s) commented on a bug without a package' %
              (bug_number, reporter))
        logging.info('LP: #%s (by %s) commented on a bug without a package' %
                     (bug_number, reporter))


def incomplete_dpkg_buffer_package(release, error, tag, since):
    '''Search and modify apport-package bugs about a release'''

    comment = "Thank you for taking the time to report this bug and helping to make Ubuntu better.  It seems that there was an error on your system when trying to install a particular package.  Please execute the following command, as it will clear your package cache, in a terminal:\n\nsudo apt-get clean\n\nThen try performing the update again.  This will likely resolve your issue, but the failure could be caused by filesystem or memory corruption.  So please also run a fsck on your filesystem(s) and a memory test.  Thanks in advance!\n\n[This is an automated message.  I apologize if it reached you inappropriately; please just reply to this message indicating so.]"
    tasks = ubuntu.searchTasks(status=['New'], order_by='-datecreated',
                               tags=['apport-package', release],
                               tags_combinator='All',
                               created_since=since)
    check = ['DpkgTerminalLog.txt', 'DpkgTerminalLog.gz',
        'VarLogDistupgradeTermlog.gz', 'VarLogDistupgradeApttermlog.gz']

    # the security team frequently adds comments saying how the bug isn't
    # a security one, so lets ignore those
    ubuntu_sec = lp.people['ubuntu-security']
    ubuntu_sec_members = ubuntu_sec.participants
    skipped_commenters = [p.name for p in ubuntu_sec_members]

    for task in tasks:
        if skip_bug(task):
            continue
        reporter = task.bug.owner.name
        skipped_commenters.append(reporter)
        bug = task.bug
        attaches = [a.title for a in bug.attachments]
        bad_attachments = ['VarLogDistupgradeAptclonesystemstate.tar.gz',
            'VarLogDistupgradeAptclonesystemstatetargz.gz',
            'VarLogDistupgradeSystemstate.tar.gz',
            'VarLogDistupgradeSystemstatetargz.gz',
            'VarLogSystemstate.tar.gz',
            'VarLogSystemstatetargz.gz']
# Disabled as every release has a security update to check for usernames and
# passwords in aptclonesystemstate
#        if set(bad_attachments).intersection(attaches):
#            if 'quantal' in bug.tags:
#                continue
#            for attachment in bug.attachments:
#                if attachment.title in bad_attachments:
#                    # the following will end up matching USERNAME:PASSWORD
#                    # which is what update-manager replaces it with
#                    # check the attachment for a username and password
#                    password_re = re.compile('/[^/@:]*:[^/@:]*@')
#                    if not error_in_attachments(task, password_re, bad_attachments):
#                        print("LP: #%s did not remove attachment %s" % (bug.id,
#                            attachment.title))
#                        logging.info("LP: #%s did not remove attachment %s" % (bug.id,
#                            attachment.title))
#                        continue
#                    else:
#                        try:
#                            attachment.removeFromBug()
#                        except errors.HTTPError:
#                            pass
#                        print("LP: #%s removed attachment %s" % (bug.id,
#                            attachment.title))
#                        logging.info("LP: #%s removed attachment %s" % (bug.id,
#                            attachment.title))
        if error_in_attachments(task, error, check) or re.search(error, task.bug.description):
            if real_comment(task, skipped_commenters) is False:
                bug = task.bug
                bug_num = bug.id
                add_tags(bug, ['%s' % tag])
                bug.newMessage(content=comment)
                bug.lp_save()
                task.status = 'Invalid'
                task.lp_save()
                print('Invalidating LP: #%s a %s bug' % (bug_num, tag))
                logging.info('LP: #%s invalidated as it is a %s bug' % (bug_num, tag))
        skipped_commenters.remove(reporter)


def apport_package_debconf_passthrough_errors(since):
    '''LP: #442941 cleanup
       This is a symptom of lots of different problems so should not be used.'''

    error1 = 'Use of uninitialized value $reply in scalar chomp at /usr/share/perl5/Debconf/FrontEnd/Passthrough.pm line 66'
    error2 = 'exit status 128'

    comment="Thank you for taking the time to report this bug and helping to make Ubuntu better. This particular bug has already been reported and is a duplicate of bug 442941, so it is being marked as such. Please look at the other bug report to see if there is any missing information that you can provide, or to see if there is a workaround for the bug.  Additionally, any further discussion regarding the bug should occur in the other report.  Please continue to report any other bugs you may find."

    # start with New moved to Confirmed
    tasks = ubuntu.searchTasks(status=['New'], order_by='-datecreated',
                               tags=['apport-package'], created_since=since)
    check = ['DpkgTerminalLog.txt', 'DpkgTerminalLog.gz']

    for task in tasks:
        if skip_bug(task):
            continue
        if error_in_attachments(task, error1, check) and error_in_attachments(task, error2, check):
            bug = task.bug
            add_tags(bug, ['debconf-passthrough'])
            mark_as_duplicate('442941', bug, comment)
            print('Marking LP: #%s as a duplicate of bug 442941' % task.bug.id)
            logging.info('LP: #%s marked as a duplicate of bug 442941' % task.bug.id)


def apport_package_segfault(since):
    comment = '''Thank you for taking the time to report this bug and helping to make Ubuntu better.  Reviewing your log files attached to this bug report it seems that a package failed to install due to a segmentation fault in the application being used for the package installation process.  Unfortunately, this bug report isn't very useful in its current state and a crash report would be much more useful.  Could you try recreating this issue by enabling apport to catch the crash report 'sudo service apport start force_start=1' and then trying to install the same package again?  This process will create a new bug report so I am marking this one as Invalid.  Thanks again for helping out!\n\n[This is an automated message.  I apologize if it reached you inappropriately; please just reply to this message indicating so.]'''
    error = re.compile('(apt| frontend|debconf|dpkg).*segfault.*\n')

    tasks = ubuntu.searchTasks(status=['New'], order_by='-datecreated',
                               tags=['apport-package'],
                               created_since=since)
    check = ['Dmesg.txt', 'CurrentDmesg.txt']

    for task in tasks:
        if skip_bug(task):
            continue
        bug_num = task.bug.id
        if error_in_attachments(task, error, check):
            bug = task.bug
            add_tags(bug, ['package-install-segfault'])
            task.status = 'Invalid'
            task.lp_save()
            bug.newMessage(content=comment)
            print("LP: #%s set to Invalid as it is a package-install-segfault" %
                bug_num)
            logging.info("LP: #%s set to Invalid as it is a package-install-segfault" %
                bug_num)


def corrupt_pyc_crashes(since):
    ''' Search for crashes due to corrupt pyc files and mark them as a
        duplicate or alert if it is with a fixed version of python '''
    db = get_crashdb(None)

    tasks = ubuntu.searchTasks(order_by='-datecreated',
                tags=['apport-crash'], created_since=since)

    for task in tasks:
        if skip_bug(task):
            continue
        bug_num = task.bug.id
        if bug_num == 1058884:
            continue
        description = task.bug.description
        if 'PythonArgs' not in description:
            continue
        try:
            report = db.download(bug_num)
        except:
            log_print('Apport issue downloading LP: #%s' % bug_num)
            continue
        interpreter = report.get('InterpreterPath', '').split('/')[-1]
        # python3.2mu
        if interpreter.endswith('mu'):
            interpreter = interpreter[0:-2]
        title = report.get('Title', '')
        # use the original title not the reporter edited one
        if 'OriginalTitle' in report:
            title = report.get('OriginalTitle', '')
        if not 'bad marshal data' in title and not 'EOF read where' in title:
            continue
        bug = task.bug
        add_tags(bug, ['pyc-corruption'])
        # 2013-05-23 need to check Dependencies.txt for python3 and python
        packages = {}
        if 'Dependencies' not in report:
            continue
        deps = report.get('Dependencies').splitlines()
        for dep in deps:
            package = dep.split(' ')[0]
            version = dep.split(' ')[1]
            packages[package] = version
        py_version = packages[interpreter]
        # fixed in versions ...
        # python2.7 => 2.7.5-1ubuntu2
        # python3 => 3.3.2-1ubuntu3
        if interpreter == 'python3':
            if apt_pkg.version_compare(py_version, '3.3.2-1ubuntu2') > 0:
                # it wasn't fixed
                log_print('LP: #%s indicates there is still pyc corruption '
                    'occuring' % bug_num)
                continue
            if apt_pkg.version_compare(py_version, '3.3.2-1ubuntu2') <= 0:
                mark_as_duplicate(1058884)
                log_print('LP: #%s re pyc-corruption marked as a dupe of 1058884' %
                    bug_num)
        elif interpreter == 'python2.7':
            if apt_pkg.version_compare(py_version, '2.7.5-1ubuntu1') > 0:
                # it wasn't fixed
                log_print('LP: #%s indicates there is still pyc corruption '
                    'occuring' % bug_num)
                continue
            if apt_pkg.version_compare(py_version, '2.7.5-1ubuntu1') <= 0:
                mark_as_duplicate(1058884)
                log_print('LP: #%s re pyc-corruption marked as a dupe of 1058884' %
                    bug_num)


def ubiquity_syslog_crashes(since):
    '''Examine syslog files for a Traceback and add it as a comment and tag
       the bug installer-crash.  Also review syslog for media errors.'''

    ubiquity = ubuntu.getSourcePackage(name='ubiquity')
    grub_installer = ubuntu.getSourcePackage(name='grub-installer')
    tasks = ubiquity.searchTasks(order_by='-datecreated',
        tags=['-installer-crash'], tags_combinator='All', created_since=since)

    pv_releases = package_version_releases('ubiquity')
    error_tags = ['hardware-error', 'squashfs-error', 'media-error']
    old_releases = ['lucid', 'maverick', 'natty', 'oneiric', 'quantal',
                    'raring', 'saucy', 'utopic', 'vivid', 'wily', 'yakkety',
                    'zesty', 'artful', 'cosmic', 'disco', 'eoan']
    EOL_RELEASE = "Thank you for taking the time to report this bug and \
helping to make Ubuntu better.  I noticed that you are installing an End of \
Life version of Ubuntu, unless you have some specific requirements to run an  \
older insecure release I recommend trying to install the latest LTS version \
of Ubuntu.  We released version 20.04, in April of 2020, and it is a Long \
Term Support release.  You can find different ways to obtain it at \
http://download.ubuntu.com.  Thanks!\n\n%s" % AUTO_RESPONSE
    P_POINT_RELEASE = "Thank you for taking the time to report this bug and \
helping to make Ubuntu better. We noticed that you were installing the LTS \
version of Ubuntu using an older version of the installation media.  The \
latest point release of 12.04 is 12.04.5. Please try installing from that \
version of the installation media and see if that resolves your issue. \
Thanks in advance!\n\n%s" % AUTO_RESPONSE
    T_POINT_RELEASE = "Thank you for taking the time to report this bug and \
helping to make Ubuntu better. We noticed that you were installing the LTS \
version of Ubuntu using an older version of the installation media.  The \
latest point release of 14.04 is 14.04.6. Please try installing from that \
version of the installation media and see if that resolves your issue. \
Thanks in advance!\n\n%s" % AUTO_RESPONSE
    X_POINT_RELEASE = "Thank you for taking the time to report this bug and \
helping to make Ubuntu better. We noticed that you were installing the LTS \
version of Ubuntu using an older version of the installation media.  The \
latest point release of 16.04 is 16.04.7. Please try installing from that \
version of the installation media and see if that resolves your issue. \
Thanks in advance!\n\n%s" % AUTO_RESPONSE
    B_POINT_RELEASE = "Thank you for taking the time to report this bug and \
helping to make Ubuntu better. We noticed that you were installing the LTS \
version of Ubuntu using an older version of the installation media.  The \
latest point release of 18.04 is 18.04.5. Please try installing from that \
version of the installation media and see if that resolves your issue. \
Thanks in advance!\n\n%s" % AUTO_RESPONSE
    F_POINT_RELEASE = "Thank you for taking the time to report this bug and \
helping to make Ubuntu better. We noticed that you were installing the LTS \
version of Ubuntu using an older version of the installation media.  The \
latest point release of 20.04 is 20.04.1. Please try installing from that \
version of the installation media and see if that resolves your issue. \
Thanks in advance!\n\n%s" % AUTO_RESPONSE
    BAD_MEDIA = "Thank you for taking the time to report this bug and \
helping to make Ubuntu better.  Reviewing your log files attached to \
this bug report it seems that there is a problem with your installation \
media (CD/DVD).  You can verify the integrity of the Ubuntu ISO files \
you downloaded by following the instructions at \
https://help.ubuntu.com/community/HowToMD5SUM.  You might also retry \
your installation with new media.  In the event that is is not in \
fact an error with your installation media please set the bug's status \
back to New.  Thanks and good luck!\n\n%s" % AUTO_RESPONSE
    # N.B. Looking at ubiquity bug reports there are very few (none?)
    # submitted about flavours
    ISOS = {
        # Focal point releases
        'Ubuntu 20.04.1 LTS "Focal Fossa" - Release amd64 (20200731)':
        '2f4250f7e09ee7947a55d3719c198933-2',
        # Focal
        'Ubuntu 20.04 LTS "Focal Fossa" - Release amd64 (20200423)':
        '262d29a6a66e8f1bebc790a773ce80ec-2',
        # Eoan
        'Ubuntu 19.10 "Eoan Ermine" - Release amd64 (20191017)':
        '5c28cc85a8264403dbac22de656da35c-2',
        # Disco
        'Ubuntu 19.04 "Disco Dingo" - Release amd64 (20190416)':
        '41abb1a0155f33e6e57269fba2f250a1-2',
        # Cosmic
        'Ubuntu 18.10 "Cosmic Cuttlefish" - Release amd64 (20181017.3)':
        'c2b268c0c79be785604aae74794a9844-2',
        # Bionic point releases
        'Ubuntu 18.04.3 LTS "Bionic Beaver" - Release amd64 (20190805)':
        '5cd23f56ff2ef60cdcb014930e37e660-2',
        'Ubuntu 18.04.2 LTS "Bionic Beaver" - Release amd64 (20190210)':
        'ea928df4b8df936361b030cd9e7b3a04-2',
        'Ubuntu 18.04.1 LTS "Bionic Beaver" - Release amd64 (20180725)':
        'a68565e5ddd372c764f556f33ff1cb86-2',
        # Bionic
        'Ubuntu 18.04 LTS "Bionic Beaver" - Release amd64 (20180426)':
        '2aa509fbbe31560badc5540a089d2b85-2',
        # Artful
        'Ubuntu 17.10 "Artful Aardvark" - Release amd64 (20180105.1)':
        '025d3ef01a56c4d3d514a60c681452c5-2',
        # Zesty
        'Ubuntu 17.04 "Zesty Zapus" - Release amd64 (20170412)':
        '37faf2e09f73d288b750b7fe001295bb-2',
        'Ubuntu 17.04 "Zesty Zapus" - Release i386 (20170412)':
        '71dba8f60b1dba5a78205a100cd9f350-2',
        'Kubuntu 17.04 "Zesty Zapus" - Release amd64 (20170412)':
        '2e799d90dee3d76f2605ae8222c1a223-2',
        'Kubuntu 17.04 "Zesty Zapus" - Release i386 (20170412)':
        '494e0ffdf54b392fcfc47e4706659c6b-2',
        'Lubuntu 17.04 "Zesty Zapus" - Release amd64 (20170412)':
        '21465fb91994f9f83cf881d42d3575f1-2',
        'Lubuntu 17.04 "Zesty Zapus" - Release i386 (20170412)':
        '0c2aa81fb456a06874e6f51932460f7b-2',
        'Ubuntu-MATE 17.04 "Zesty Zapus" - Release amd64 (20170412)':
        'f44c40e99234b0341d19ed79706f071d-2',
        'Ubuntu-MATE 17.04 "Zesty Zapus" - Release i386 (20170412)':
        'c369266ea766c5feec4408025b44a57d-2',
        'Xubuntu 17.04 "Zesty Zapus" - Release amd64 (20170412)':
        '2067c7a114550ff522f922eee194e71a-2',
        'Xubuntu 17.04 "Zesty Zapus" - Release i386 (20170412)':
        '944db3c99d13bc484ac0f7f31a2eeefa-2',
        # Xenial point release
        'Ubuntu 16.04.3 LTS "Xenial Xerus" - Release amd64 (20170801)':
        '482cd2e33d77363147f1df9d7da60e66-2',
        'Ubuntu 16.04.3 LTS "Xenial Xerus" - Release i386 (20170801)':
        '9d25bb517137c3aa135c21d6be05f2eb-2',
        'Kubuntu 16.04.3 LTS "Xenial Xerus" - Release amd64 (20170801)':
        '49565f0969df51934d69893acceebb16-2',
        'Kubuntu 16.04.3 LTS "Xenial Xerus" - Release i386 (20170801)':
        'f83cb38c724a0861e49770834639a59c-2',
        'Lubuntu 16.04.3 LTS "Xenial Xerus" - Release amd64 (20170801)':
        'ef4696f2f8bacaf5ec13595ea8b507ac-2',
        'Lubuntu 16.04.3 LTS "Xenial Xerus" - Release i386 (20170801)':
        'c7494283f722ff2d40993f6d4828d50b-2',
        'Ubuntu-MATE 16.04.3 LTS "Xenial Xerus" - Release amd64 (20170801)':
        '47673a25c952e833c3fb984272ac9316-2',
        'Ubuntu-MATE 16.04.3 LTS "Xenial Xerus" - Release i386 (20170801)':
        '4b8f3d1ff623d2490b8e6f81bfad8f16-2',
        'Mythbuntu 16.04.3 LTS "Xenial Xerus" - Release amd64 (20170801)':
        '110622ea038f5c88e763b3bff084f142-2',
        'Mythbuntu 16.04.3 LTS "Xenial Xerus" - Release i386 (20170801)':
        '1565fb5b1e7ef858ad618e3640518a34-2',
        'Xubuntu 16.04.3 LTS "Xenial Xerus" - Release amd64 (20170801)':
        '89b2d2604f1721c763666e6b5b109a15-2',
        'Xubuntu 16.04.3 LTS "Xenial Xerus" - Release i386 (20170801)':
        '90f64e3df57e63c129aad5a60a7a6684-2',
        # Xenial
        'Ubuntu 16.04 LTS "Xenial Xerus" - Release amd64 (20160420.1)':
        '1a235e539972579c22a0bec9a077f7c3-2',
        'Ubuntu 16.04 LTS "Xenial Xerus" - Release i386 (20160420.1)':
        'a416d67e2dccd1e5584b23ccfd20997e-2',
        'Kubuntu 16.04 LTS "Xenial Xerus" - Release amd64 (20160420.1)':
        'f6839c48f3f4c78d6f0c0f81d5f7bfb9-2',
        'Kubuntu 16.04 LTS "Xenial Xerus" - Release i386 (20160420.1)':
        'a415817552fce7ca3725b21595ec988f-2',
        'Lubuntu 16.04 LTS "Xenial Xerus" - Release amd64 (20160420.1)':
        '934c7b999d06f953143c3760db5c0f92-2',
        'Lubuntu 16.04 LTS "Xenial Xerus" - Release i386 (20160420.1)':
        '973cda5dcbb1359e40135dcc77c391dd-2',
        'Mythbuntu 16.04 LTS "Xenial Xerus" - Release amd64 (20160420.1)':
        '95abdcda290b30e57bfab69349aad690-2',
        'Mythbuntu 16.04 LTS "Xenial Xerus" - Release i386 (20160420.1)':
        'd838a75c6f79ab5551cf7caed957331e-2',
        'Ubuntu-MATE 16.04 LTS "Xenial Xerus" - Release amd64 (20160420.1)':
        '8cd7575d6522a4115dc106e797018dbb-2',
        'Ubuntu-MATE 16.04 LTS "Xenial Xerus" - Release i386 (20160420.1)':
        '21db08dca1e052609c0e9b8a04c323b4-2',
        'Xubuntu 16.04 LTS "Xenial Xerus" - Release amd64 (20160420.1)':
        '680bcc422c1bc73a69e015b47c6250b8-2',
        'Xubuntu 16.04 LTS "Xenial Xerus" - Release i386 (20160420.1)':
        '8fe812727c4df456a15c73ff60cb3f87-2',
        # Trusty 5th Point Release
        'Kubuntu 14.04.5 LTS "Trusty Tahr" - Release amd64 (20160803)':
        'dc959edaf0e7481839fef518fa4c3925-2',
        'Kubuntu 14.04.5 LTS "Trusty Tahr" - Release i386 (20160803)':
        'd47f5dc69a3013336c3d4c55ea9977ac-2',
        'Lubuntu 14.04.5 LTS "Trusty Tahr" - Release amd64 (20160803)':
        '08f27c7b67f94c15a0202d83fb7db56c-2',
        'Lubuntu 14.04.5 LTS "Trusty Tahr" - Release i386 (20160803)':
        '2122fb2321a932db0170525ba6f75755-2',
        'Mythbuntu 14.04.5 LTS "Trusty Tahr" - Release amd64 (20160804)':
        '15ad2dee89ce479cad28639796fa0e05-2',
        'Mythbuntu 14.04.5 LTS "Trusty Tahr" - Release i386 (20160804)':
        'd3ab8f3dd598b08e28997dffd2844782-2',
        'Ubuntu 14.04.5 LTS "Trusty Tahr" - Release amd64 (20160803)':
        '13ca05a159a1bdb82d0b43ecb72b49b0-2',
        'Ubuntu 14.04.5 LTS "Trusty Tahr" - Release i386 (20160803)':
        '7a5252d303c1fc7e6c0edd34829774a2-2',
        'Xubuntu 14.04.5 LTS "Trusty Tahr" - Release amd64 (20160803)':
        '8f272e79063d78f7542f8bb66d34dba1-2',
        'Xubuntu 14.04.5 LTS "Trusty Tahr" - Release i386 (20160803)':
        'aedfed14a2a5e72bd8c63a9dc355fcee-2',
        # Trusty
        'Ubuntu 14.04 LTS "Trusty Tahr" - Release amd64 (20140417)':
        'bac8495d27ebe9f23250907977f69e73-2',
        'Ubuntu 14.04 LTS "Trusty Tahr" - Release i386 (20140417)':
        'e34e567007c885a0f73c7c604c8983d6-2',
        'Kubuntu 14.04 LTS "Trusty Tahr" - Release amd64 (20140416.1)':
        'a15b340150eaddc2c781c764dd4be3b0-2',
        'Kubuntu 14.04 LTS "Trusty Tahr" - Release i386 (20140416.1)':
        'c8e96c71b8174204439714f22f6d6773-2',
        'Lubuntu 14.04 LTS "Trusty Tahr" - Release amd64 (20140416.2)':
        'f29fac6975aa3eda697d40a855a00ed2-2',
        'Lubuntu 14.04 LTS "Trusty Tahr" - Release i386 (20140416.2)':
        '23b8f396b23696c2be9993a596fb1397-2',
        'Xubuntu 14.04 LTS "Trusty Tahr" - Release amd64 (20140416.2)':
        'ac82c0be43480770337bb3ab8da0d3cb-2',
        'Xubuntu 14.04 LTS "Trusty Tahr" - Release i386 (20140416.2)':
        '3c937a1feef34fc49d1b5821b7321474-2',
    }

    for task in tasks:
        tag_test_case(task)
        if skip_bug(task):
            continue
        bug = task.bug
        bug_num = bug.id
        # ubiquity auto tags bugs ubiquity- so check to see if crichton
        # commented
        commenters = set([message.owner.name for message in bug.messages])
        if 'crichton' in commenters:
            continue
        description = bug.description
        persistent = False
        if 'InstallationMedia' in description or 'LiveMediaBuild' in description:
            has_casper = False
            media = ''
            for attachment in bug.attachments:
                if attachment.title == 'Casper.txt':
                    has_casper = True
            for line in description.split('\n'):
                if 'InstallCmdLine:' in line and 'persistent' in line:
                    persistent = True
                if 'InstallationMedia: ' in line or 'LiveMediaBuild' in line:
                    media = ' '.join(line.split(' ')[1:])
            if 'Ubuntu 12.04 LTS' in media or \
                    'Ubuntu 12.04.1' in media or \
                    'Ubuntu 12.04.2' in media or \
                    'Ubuntu 12.04.3' in media or \
                    'Ubuntu 12.04.4' in media:
                bug.newMessage(content=P_POINT_RELEASE)
                task.status = 'Incomplete'
                task.lp_save()
                log_print('LP: #%s recommended to try new 12.04.5 ' \
                    'point release' % bug_num)
                continue
            if 'Ubuntu 14.04 LTS' in media or \
                    'Ubuntu 14.04.1' in media  or \
                    'Ubuntu 14.04.2' in media or \
                    'Ubuntu 14.04.3' in media or \
                    'Ubuntu 14.04.4' in media:
                bug.newMessage(content=T_POINT_RELEASE)
                task.status = 'Incomplete'
                task.lp_save()
                log_print('LP: #%s recommended to try new 14.04.5 ' \
                    'point release' % bug_num)
                continue
            if 'Ubuntu 16.04 LTS' in media or \
                    'Ubuntu 16.04.1' in media or \
                    'Ubuntu 16.04.2' in media:
                bug.newMessage(content=X_POINT_RELEASE)
                task.status = 'Incomplete'
                task.lp_save()
                log_print('LP: #%s recommended to try new 16.04.2 ' \
                    'point release' % bug_num)
                continue
            if media in ISOS and has_casper:
                key = ISOS[media]
                if not error_in_attachments(task,
                        key, 'Casper.txt'):
                    if persistent:
                        log_print("LP: #%s ident-mismatch but " \
                            "persistent" % bug_num)
                        continue
                    if task.status == 'Incomplete':
                        continue
                    bug.newMessage(content=BAD_MEDIA)
                    task.status = 'Incomplete'
                    task.lp_save()
                    add_tags(bug, ['ident-mismatch'])
                    log_print('LP: #%s did not match the iso ident'
                              % bug_num)
                    continue
        version = None
        for attachment in bug.attachments:
            # reload the tags after every run
            tags = bug.tags
            traceback = ''
            contents = b''
            # people name attachments oddly so check them all
            if attachment.title in ['Dependencies.txt', 'Traceback.txt',
                'ProcMaps.txt', 'ProcStatus.txt', 'UbiquityDm.txt',
                'Casper.txt', 'UbiquityDebug.txt', 'UbiquityPartman.txt']:
                continue
            elif attachment.title.endswith('.gz'):
                try:
                    hosted_file_buffer = read_gz_file(attachment, task)
                except OSError as error:
                    if error.strerror == 'Connection refused':
                        continue
                except ServerError:
                    continue
            else:
                hosted_file = attachment.data
                try:
                    hosted_file_buffer = hosted_file.open()
                # sometimes attachments don't exist
                except AttributeError:
                    continue
                except OSError as error:
                    if error.strerror == 'Connection refused':
                        continue
                except ServerError:
                    continue
            for line in hosted_file_buffer:
                match = re.search(b'ubiquity.*Ubiquity (\d\.?)+', line)
                if match:
                    version = match.group(0).split()[-1]
                contents += line
            if version is None:
                break
            try:
                release_tag = pv_releases[version]
            except KeyError:
                release_tag = None
            v_tag = 'ubiquity-%s' % version
            t_notice = None
            if v_tag not in tags:
                add_tags(bug, ['%s' % v_tag])
                t_notice = 'LP: #%s tagged %s' % (bug_num, v_tag)
            if release_tag and release_tag not in tags:
                add_tags(bug, ['%s' % release_tag])
                if t_notice:
                    t_notice += ' and %s' % release_tag
                else:
                    t_notice = 'LP: #%s tagged %s' % (bug_num, release_tag)
                break
            if t_notice:
                print('%s' % t_notice)
                logging.info('%s' % t_notice)
            if release_tag in old_releases:
                bug.newMessage(content=EOL_RELEASE)
                task.status = 'Invalid'
                task.lp_save()
                print('LP: #%s recommended to try focal' % bug_num)
                logging.info('LP: #%s recommended to try focal' % bug_num)
                break
            # look for ubiquity-upgrade bug reports
            if re.search('AptDaemon: INFO: UpgradeSystem() was called with', contents):
                add_tags(bug, ['ubiquity-upgrade'])
            # media errors
            if re.search('Attached .* CD-ROM (.*)', contents):
                if set(error_tags).intersection(tags):
                    continue
                if not re.search('Attached .* CD-ROM (\w+)', contents):
                    continue
                cd_drive = re.search('Attached .* CD-ROM (\w+)',
                    contents).group(1)
                cd_error = re.search('Buffer I/O error on device %s' % cd_drive,
                    contents)
                if cd_error:
                    media_comment = "Thank you for taking the time to report this bug and helping to make Ubuntu better.  Reviewing your log files attached to this bug report it seems that there is a problem with your installation media (CD/DVD).  Please retry your installation with new media and if it still fails I'd recommend testing your CD drive with other media.  In the event that is is not in fact an error with your installation media please set the bug's status back to New.  Thanks and good luck!\n\n[This is an automated message.  I apologize if it reached you inappropriately; please just reply to this message indicating so.]"
                    add_tags(bug, ['media-error'])
                    bug.newMessage(content=media_comment,
                        subject='Media Error')
                    bug.lp_save()
                    task.status = 'Invalid'
                    task.lp_save()
                    bug_number = task.bug.id
                    print('LP: #%s set to Invalid as it is a media-error bug' %
                        bug_number)
                    logging.info('LP: #%s set to Invalid as it is a media-error bug' %
                        bug_number)
                    break
            # hard disk errors
            # 2011-11-22 - this won't catch everything but seems like a good
            # start
            if re.search('I/O error, dev sda', contents):
                if set(error_tags).intersection(tags):
                    continue
                hardware_comment = "Thank you for taking the time to report this bug and helping to make Ubuntu better.  Reviewing your log files attached to this bug report it seems that there is a problem with the disk to which you are trying to install Ubuntu.  I'd recommend performing a back up of your existing operating system and then investigating the situation.  Measures you might take include checking cable connections for your disks and using software tools to investigate the health of your hardware.  In the event that is is not in fact an error with your hardware please set the bug's status back to New.  Thanks and good luck!\n\n%s" % AUTO_RESPONSE
                add_tags(bug, ['hardware-error'])
                bug.newMessage(content=hardware_comment,
                    subject='Hardware Error')
                bug.lp_save()
                task.status = 'Invalid'
                task.lp_save()
                bug_number = task.bug.id
                print('LP: #%s set to Invalid as it is a hardware-error bug' %
                    bug_number)
                logging.info('LP: #%s set to Invalid as it is a hardware-error bug' %
                    bug_number)
                break
            match = re.search('Exception during installation(.*)(python|plugininstall\.py): \n',
                contents, re.DOTALL)
            if match:
                traceback = match.group(0)
            else:
                continue

            if len(traceback) >= 1:
                bug_num = bug.id
                release_tags = set(RELEASE_TAGS).intersection(tags)
                r_tags = ''
                if release_tags is not None:
                    r_tags = ' '.join(release_tags)
                print('LP: #%s install failure re %s' % (bug_num,
                    r_tags))
                logging.info('LP: #%s install failure re %s' % (bug_num,
                    r_tags))
                add_tags(bug, ['installer-crash'])
                # OMG WTF
                merge_fixed = {'quantal', 'raring', 'saucy', 'trusty'}
                if 'Problem with MergeList' in traceback and \
                        not merge_fixed.intersection(tags):
                    task.status = 'Confirmed'
                    task.lp_save()
                    mark_as_duplicate('346386', bug)
                    log_print('LP: #%s treated as a merge-list-bug' %
                        bug_num)
                    break
                elif 'Problem with MergeList' in traceback and \
                        merge_fixed.intersection(tags):
                    print('LP: #%s re %s has a merge-list issue' %
                        (bug_num, r_tags))
                    logging.info('LP: #%s re %s has a merge-list issue' %
                        (bug_num, r_tags))
                comment = ''
                for line in traceback.split('\n'):
                    comment += '%s\n' % line
                    # because the re is greedy and might catch multiple
                    # crashes just use the first one
                    if line.endswith('plugininstall.py: '):
                        break
                bug.newMessage(content=comment, subject='Traceback')
                if 'vnc.create_password(passwd)' in comment and \
                        'Password should be passed as bytes' in comment:
                    mark_as_duplicate('1290460', bug)
                    log_print("LP: #%s mythbuntu vnc password issue" %
                        bug_num)
                    break
                # set the importance of crashes to high
                if 'trusty' in r_tags or 'precise' in r_tags:
                    task.importance = 'High'
                    task.lp_save()
                    print('LP: #%s was set to High' % (bug.id))
                    logging.info('LP: #%s was set to High' % (bug.id))
                    # 2012-01-24 this doesn't seem to be working in all cases see
                    # LP: #916837
                    # 2012-05-18 refresh the task in case that is the issue
                    task.lp_refresh()
                if 'GrubInstaller failed with' in traceback:
                    task.target == grub_installer
                    task.lp_save()
                    print('LP: #%s reassigned to grub-installer' % bug.id)
                    logging.info('LP: #%s reassigned to grub-installer' %
                        bug.id)
                continue

def apport_package_dupe_sig_fixer(since):
    '''The method for creating a DuplicateSignature for apport-package bug
       reports created a signature that was too short. (LP: #1692127) This
       checks bug reports from systems without the updated version of apport
       and fixes the DuplicateSignature. Additionally, if the bug report is a
       duplicate of another bug the new signatures for both bugs should be
       compared so we can confirm they are really duplicates.'''

    # what we'll say when modifying the bug
    comment = '''Due to a bug in apport's method for creating a duplicate \
signature, the DuplicateSignature in this report was incorrect. We are \
fixing that now, sorry for the noise!'''

    # setup the apport database
    db = get_crashdb(None)

    tasks = []
    # this only affects 16.04, 16.10, and 17.04
    for series in ['xenial', 'yakkety', 'zesty']:
        tasks.extend([task for task in
                     ubuntu.searchTasks(tags=['apport-package', series],
                                        tags_combinator='All',
                                        order_by='-datecreated',
                                        created_since=since,
                                        omit_duplicates=False)])

    prog = re.compile(rb"""^(Authenticating|
                            De-configuring|
                            Examining|
                            Installing|
                            Preparing|
                            Processing\ triggers|
                            Purging|
                            Removing|
                            Replaced|
                            Replacing|
                            Setting\ up|
                            Unpacking|
                            Would remove).*
                           \.\.\.\s*$""", re.X)

    bad_count = 0
    good_count = 0

    for task in tasks:
        if skip_bug(task):
            continue
        bug_num = task.bug.id
        bug = task.bug
        try:
            report = db.download(bug_num)
        except:
            log_print('Apport issue downloading LP: #%s' % bug_num)
            continue
        # check the apport version
        apport_vers = report.get('ApportVersion', '')
        # 2017-06-13 need to put in version numbers that are fixed
        if apport_vers == 'fixed version':
            continue
        current_dupe_sig = report.get('DuplicateSignature', '')
        if not current_dupe_sig:
            continue
        if len(current_dupe_sig.split('\n')) == 1:
            log_print("Skipping one line DupeSig for LP: #%s" % bug_num)
            continue
        if prog.match(current_dupe_sig.split('\n')[1]):
            good_count += 1
            continue
        else:
            log_print("DupeSig is short for LP: #%s" % bug_num)
            bad_count += 1
            dupe_sig = ''
            dupe_sig_created = False
            for attachment in bug.attachments:
                if attachment.title == 'DpkgTerminalLog.txt.gz':
                    log_print("Found a bug with gz'ed DpkgTerminalLog LP: #%s" %
                          bug_num)
            # 2017-06-13 does this work with .gz files?
            dpkg_term_log = report.get('DpkgTerminalLog', '')
            if not dpkg_term_log:
                continue
            for line in dpkg_term_log.split('\n'):
                if prog.search(line):
                    # just use the first line from the current_dupe_sig rather
                    # than recreating it, since nothing is wrong with it.
                    dupe_sig = '%s\n' % current_dupe_sig.split('\n')[0]
                    dupe_sig += '%s\n' % line
                    dupe_sig_created = True
                    continue
                dupe_sig += '%s\n' % line
                if 'dpkg: error' in dupe_sig and line.startswith(' '):
                    if dupe_sig_created:
                        break
            report['DuplicateSignature'] = dupe_sig
            # remove the attachment first
            for attachment in bug.attachments:
                if attachment.title == 'DuplicateSignature.txt':
                    try:
                        attachment.removeFromBug()
                    except errors.HTTPError:
                        pass # LP: #249950 workaround
                    break
            # add the new DuplicateSignature to bug in Launchpad
            db.update(bug_num, report, '', change_description=False,
                      attachment_comment=comment,
                      key_filter=['DuplicateSignature'])
            log_print("Fixed short DupeSig in LP: #%s" % bug_num)
            continue

    #print("Good count: %i" % good_count)
    #print("Bad count: %i" % bad_count)


def apport_package_hardware_error(since, bug_numbers=None):
    '''Examine dmesg to see if there is a hardware error with the device
       holding the /, /var or /usr mount point.'''

    comment = "Thank you for taking the time to report this bug and helping to make Ubuntu better.  Reviewing your dmesg attachment in this bug report it seems that there is a problem with your hardware.  I recommend performing a back up and then investigating the situation.  Measures you might take include checking cable connections and using software tools to investigate the health of your hardware.  In the event that is is not in fact an error with your hardware please set the bug's status back to New.  Thanks and good luck!\n\n[This is an automated message.  I apologize if it reached you inappropriately; please just reply to this message indicating so.]"

    # because you can't add a bug task to a collection afaict
    tasks = [task for task in ubuntu.searchTasks(tags=['apport-package'],
        order_by='-datecreated', created_since=since)]
    if bug_numbers:
        for bug_number in bug_numbers:
            bug = lp.bugs[bug_number]
            tasks.append(bug.bug_tasks[0])
    for task in tasks:
        if skip_bug(task):
            continue
        if 'hardware-error' in task.bug.tags:
            continue
        devs_to_check = []
        error_found = False
        bug = task.bug
        for attachment in bug.attachments:
            if attachment.title == 'Df.txt':
                hosted_file = attachment.data
                hosted_file_buffer = hosted_file.open()
                for line in hosted_file_buffer:
                    line = line.strip(b'\n')
                    if line.endswith(b'/'):
                        # without manipulation it'd look like /dev/sda1
                        root_device = line.split(b' ')[0].strip(b'0123456789')
                        root_device = root_device.replace(b'/dev/', b'')
                        devs_to_check.append(root_device)
                    elif line.endswith(b'/usr'):
                        # without manipulation it'd look like /dev/sda1
                        usr_device = line.split(b' ')[0].strip(b'0123456789')
                        usr_device = usr_device.replace(b'/dev/', b'')
                        devs_to_check.append(usr_device)
                    elif line.endswith(b'/var'):
                        # without manipulation it'd look like /dev/sda1
                        var_device = line.split(b' ')[0].strip(b'0123456789')
                        var_device = var_device.replace(b'/dev/', b'')
                        devs_to_check.append(var_device)
            elif attachment.title == 'Dmesg.txt':
                hosted_file = attachment.data
                hosted_file_buffer = hosted_file.open()
                for line in hosted_file_buffer:
                    line = line.strip(b'\n')
                    if b'I/O error' in line:
                        # no device in this line
                        if b'journal commit I/O error' in line:
                            continue
                        for dev in devs_to_check:
                            if re.search(dev, line):
                                error_found = True
                if error_found:
                    add_tags(bug, ['hardware-error'])
                    bug.newMessage(content=comment)
                    bug.lp_save()
                    task.status = 'Invalid'
                    task.importance = 'Low'
                    task.lp_save()
                    print('LP: #%s set to Invalid as it has a hardware error' % bug.id)
                    logging.info('LP: #%s set to Invalid as it has a hardware error' % bug.id)
                    break


def apport_package_update_initramfs(release, since):
    '''Examine package install failures to determine if they are due to having
    an old version of update-initramfs on a LiveMediaBuild of Ubuntu'''
    # To be safe, since we don't know what uses update-initramfs, we will use
    # a package whitelist
    SRC_PKGS = ['initramfs-tools', 'bcmwl', 'nvidia-graphics-drivers',
        'fuse', 'nvidia-graphics-drivers-173', 'fglrx-installer', 'watershed',
        'nvidia-graphics-drivers-96']
    comment = 'Thank you for taking the time to report this bug and helping to make Ubuntu better.  The version of update-initramfs that you likely have installed is unable to work on a persistent usb stick.  You can resolve this issue by upgrading the version of casper on your system. This can be done via the commands:\n\nsudo apt-get update; sudo apt-get install casper\n\nThen performing the software update or installation that you were originally trying.  If this does not resolve your bug please set its status back to New.  Thanks in advance!'
    tasks = ubuntu.searchTasks(tags=['apport-package', release],
        tags_combinator='All', order_by='-datecreated', created_since=since)
    for task in tasks:
        if skip_bug(task):
            continue
        bug = task.bug
        if 'needs-new-casper' in bug.tags:
            continue
        if 'precise' in bug.tags:
            continue
        if not 'LiveMediaBuild' in bug.description:
            continue
        if not 'post-installation' in bug.title:
            continue
        #if not 'exit status 1' in bug.title:
        #    continue
        # skip Df.txt with /target because this mount is created by ubiquity
        if error_in_attachments(task, '100% /cdrom', 'Df.txt') and \
            not error_in_attachments(task, '/target', 'Df.txt'):
            full_live_media(task)
            continue
        if error_in_attachments(task, 'failed to detect canonical device of overlayfs', 'DpkgTerminalLog.txt'):
            # something else is really wrong here
            continue
        if task.bug_target_display_name.split(' ')[0] not in SRC_PKGS:
            print('Skipping LP: #%s because its task is about %s' %
                (task.bug.id, task.bug_target_display_name))
            logging.info('LP: #%s not update-initramfs commented because its task is about %s' %
                (task.bug.id, task.bug_target_display_name))
            continue
        add_tags(bug, ['needs-new-casper'])
        bug.newMessage(content=comment)
        bug.lp_save()
        task.status = 'Invalid'
        task.lp_save()
        print('LP: #%s set to Invalid as it needs a new update-initramfs' %
            task.bug.id)
        logging.info('LP: #%s set to Invalid as it needs a new update-initramfs' %
            task.bug.id)


def subscribe_teams_to_patched_bugs(since):
    FB_PACKAGES = [p.name for p in FBUGS.getBugSubscriberPackages()]
    tasks = ubuntu.searchTasks(modified_since=since,
        tags='-needs-packaging', has_patch=True)
    ubuntu_dev = lp.people['ubuntu-dev']
    ubuntu_dev_members = ubuntu_dev.participants
    reviewers = lp.people['ubuntu-reviewers']
    sponsors = lp.people['ubuntu-sponsors']
    sec_sponsors = lp.people['ubuntu-security-sponsors']
    kern_triage = lp.people['terrykrudd']
    # teams that process bugs with patches
    workflow_teams = ['ubuntu-sru', 'ubuntu-mir', 'ubuntu-release',
        'ubuntu-reviewers', 'motu-sru', 'ubuntu-security-sponsors',
        'ubuntu-sponsors']
    # tags that indicate a review has happened
    reviewed_tags = ['patch-needswork', 'patch-forwarded-upstream',
        'patch-forwarded-debian', 'indicator-application',
        'patch-accepted-upstream', 'patch-accepted-debian',
        'patch-rejected-upstream', 'patch-rejected-debian', 'patch-rejected']
    # 2023-08-11 These packages presumably have their own workflow but the
    # original reason for this escapes me.
    special_pkgs = ['ubuntu-docs (Ubuntu)', 'xubuntu-docs (Ubuntu)']
    for task in tasks:
        if skip_bug(task):
            continue
        bug = task.bug
        bug_number = task.bug.id
        #print("LP: #%s" % bug_number)
        history = [a.whatchanged for a in bug.activity]
        if 'removed subscriber %s' % sponsors.display_name in history:
            continue
        if 'removed subscriber %s' % reviewers.display_name in history:
            continue
        if 'removed subscriber %s' % sec_sponsors.display_name in history:
            continue
        bug_tags = bug.tags
        if bug.private:
            print('Skipping LP: #%s because it is private' % bug.id)
            continue
        if set(reviewed_tags).intersection(bug_tags):
            continue
        subscribers = []
        try:
            for subscription in bug.subscriptions:
                subscriber = subscription.person
                try:
                    if subscriber.private:
                        continue
                except ClientError as error:
                    if error.response['status'] == '410':
                        continue
                subscribers.append(subscriber.name)
        except errors:
            print('LP: #%s had: %s' % (bug.id, errors))
            continue
        # don't act on sync requests with the archive admin team subscribed
        if 'sync' in bug.title.lower() and 'ubuntu-archive' in subscribers:
            continue
        # don't act on bugs that already have a sponsor's team subscribed
        if set(workflow_teams).intersection(subscribers):
            continue
        for attachment in bug.attachments:
            if not attachment.type == 'Patch':
                continue
            # depending on the queue's throughput we'll want to start adding historical bugs too
            try:
                if not attachment.message.date_created.strftime('%Y-%m-%d') > '2009-12-24':
                    print('Skipping LP: #%s because the patch is too old' % bug_number)
                    logging.info('LP: #%s skipped because the patch is too old' % bug_number)
                    continue
            except NotFound:
                print('Skipping LP: #%s due to inability to find date attachment added' % bug_number)
                continue
            target_name = task.bug_target_name
            # The Canonical OEM team has their own sponsorship process so
            # don't autosubscribe ubuntu-sponsors
            oem_project = False
            for task in bug.bug_tasks:
                if task.bug_target_name == 'oem-priority':
                    oem_project = True
                    break
            if target_name in special_pkgs or target_name.startswith('oem-') \
                    or oem_project:
                if oem_project:
                    target_name = 'oem-priority'
                else:
                    target_name = task.bug_target_name.split(' ')[0]
                print('Skipping LP: #%s because it is a special package %s' %
                      (bug_number, target_name))
                logging.info(
                    'LP: #%s skipped because it is about special package %s' %
                    (bug_number, target_name))
                continue
            if 'patch' not in bug_tags:
                add_tags(bug, ['patch'])
            if re.search('^linux.*Ubuntu\)$', target_name):
                if kern_triage.name not in subscribers:
                    bug.subscribe(person=kern_triage)
                    log_print('LP: #%s had kern triager subscribed as it has a patch %s attached' % (bug.id,
                        attachment.title))
                break
            # check to see if the attachment owner can upload the patch
            owner = attachment.message.owner
            if owner in ubuntu_dev_members:
                print('Skipping LP: #%s because the patch owner is in ubuntu-dev' % bug_number)
                logging.info('LP: #%s with %s skipped because the patch owner is in ubuntu-dev' % (bug_number, attachment.title))
                break
            # check the attachment contents to see if it is a debdiff by
            # looking for debian/changelog
            if attachment.title.endswith('.gz'):
                if read_gz_file(attachment, task):
                    hosted_file_buffer = read_gz_file(attachment, task)
                else:
                    break
            else:
                hosted_file = attachment.data
                hosted_file_buffer = hosted_file.open()
            debdiff = False
            for line in hosted_file_buffer:
                if b'debian/changelog' in line:
                    debdiff = True
                    break
            src_pkg = target_name.split(' ')[0]
            if debdiff:
                comment = '''The attachment "%s" seems to be a debdiff.  The ubuntu-sponsors team has been subscribed to the bug report so that they can review and hopefully sponsor the debdiff.  If the attachment isn't a patch, please remove the "patch" flag from the attachment, remove the "patch" tag, and if you are member of the ~ubuntu-sponsors, unsubscribe the team.\n\n[This is an automated message performed by a Launchpad user owned by ~brian-murray, for any issue please contact him.]''' % (attachment.title)
                if bug.security_related:
                    bug.subscribe(person=sec_sponsors)
                else:
                    bug.subscribe(person=sponsors)
                bug.newMessage(content=comment)
                print('LP: #%s re %s%s has a debdiff (%s)' % (bug.id,
                    src_pkg, ('', ' (foundations)')[src_pkg in FB_PACKAGES], attachment.title))
                logging.info('LP: #%s had sponsors team subscribed as it has debdiff %s attached' % (bug.id,
                    attachment.title))
                break
            else:
                comment = '''The attachment "%s" seems to be a patch.  If it isn't, please remove the "patch" flag from the attachment, remove the "patch" tag, and if you are a member of the ~ubuntu-reviewers, unsubscribe the team.\n\n[This is an automated message performed by a Launchpad user owned by ~brian-murray, for any issues please contact him.]''' % (attachment.title)
                try:
                    atitle = '%s' % attachment.title
                except UnicodeDecodeError:
                    atitle = 'unknown'
                print("LP: #%s re %s%s has patch %s attached - enqueued" % (bug.id,
                    src_pkg, ('', ' (foundations)')[src_pkg in FB_PACKAGES], atitle))
                logging.info("LP: #%s had reviewers team subscribed as it has patch %s attached" % (bug.id,
                    atitle.encode('utf-8')))
                bug.subscribe(person=reviewers)
                bug.newMessage(content=comment)
                break


def package_manager_triaging(since, bug_numbers=None):
    '''Perform auto triage of common package manager bugs'''

    MERGE_LIST_COMMENT = "Thank you for taking the time to report this bug and helping to make Ubuntu better.  Reviewing your bug report it seems that you are experiencing bug 346386.  This issue happens when you have tried to update your package information when you are a network that didn't return the right files.  This issue can be cleaned up by using the following commands in a terminal:\n\n1.) sudo rm /var/lib/apt/lists/*\n2.) sudo apt-get update\n\nThanks and good luck!\n\n[This is an automated message.  I apologize if it reached you inappropriately; please reopen the bug task if it was incorrect.]"

    # these could appear in multiple packages so just all new bug tasks
    tasks = [task for task in ubuntu.searchTasks(order_by='-datecreated',
        status=['New'], created_since=since)]
    # because you can't add a bug task to a collection afaict
    if bug_numbers:
        for bug_number in bug_numbers:
            bug = lp.bugs[bug_number]
            tasks.append(bug.bug_tasks[0])
    for task in tasks:
        if skip_bug(task):
            continue
        if 'bot-comment' in task.bug.tags:
            continue
        bug = task.bug
        description = bug.description
        title = bug.title
        merge_re = 'Encountered a section with no Package: header.*Problem with MergeList'
        if re.search(merge_re, description) or re.search(merge_re, title):
            bug_num = bug.id
            mark_as_duplicate('346386', bug, comment=MERGE_LIST_COMMENT)
            add_tags(bug, ['bot-comment'])
            print('LP: #%s marked as a duplicate of bug 346386 as it is a MergeList issue') % bug_num
            logging.info('LP: #%s marked as a duplicate of bug 346386 as it is a MergeList issue' % bug_num)


def sru_verification_nag():
    import urllib.request, urllib.parse, urllib.error
    from bs4 import BeautifulSoup

    # create a list of people for whom comments will be ignored when
    # displaying the last comment in the report
    ignored_commenters = []
    ubuntu_sru = lp.people['ubuntu-sru']
    for participant in ubuntu_sru.participants:
        ignored_commenters.append(participant)
    ignored_commenters.append(lp.people['janitor'])
    ignored_commenters.append(
        lp.people['bug-watch-updater'])

    url = 'http://people.canonical.com/~ubuntu-archive/pending-sru.html'
    report_page = urllib.urlopen(url)
    report_contents = report_page.read()
    soup = BeautifulSoup(report_contents)
    srus = {}
    tables = soup.findAll("table")
    # table 6 is the upload queue status
    # this is dumb but the tables have no metadata (title or class) in them
    for table in tables[0:4]:
        release = table.previous.previous
        if release == 'Upload queue status at a glance:':
            continue
        srus[release] = {}
        trs = table.findAll("tr")
        for tr in trs:
            cols = tr.findAll("td")
            if len(cols) == 0:
                continue
            # has it been awaiting verification for 90 days?
            if int(cols[5].string) > 90:
                pkg_link = cols[3].findChild('a')['href']
                links = cols[4].findChildren('a')
                bugs = [link['href'].split('/')[-1] for link in links]
                srus[release][pkg_link] = bugs
    for sru in srus:
        release = sru
        for package in srus[release]:
            pkg_name = package.split('/')[-2]
            for sru_bug in srus[release][package]:
                bug = lp.bugs[sru_bug]
                tags = bug.tags
                if ('kernel-tracking-bug' in tags or
                    'kernel-release-tracking-bug' in tags):
                    continue
                if ('verification-failed' in tags or
                    'verification-failed-%s' % release in tags):
                    continue
                elif (('verification-done' in tags or
                       'verification-done-%s' % release in tags) and
                       'verification-needed' in tags):
                    # what to do here?
                    continue
                elif ('verification-done' in tags or
                      'verification-done-%s' % release in tags):
                    continue
                # XXX: 2012-11-15 really check if there are no comments after removal-candidate tag was added
                elif ('removal-candidate' in tags or
                      'bot-stop-nagging' in tags):
                    continue
                else:
                    try:
                        last_message_dc = bug.date_last_message
                        last_message_date = last_message_dc.replace(
                            minute=0, second=0, microsecond=0)
                        pkg_page = urllib.urlopen(package)
                        pkg_contents = pkg_page.read()
                        soup = BeautifulSoup(pkg_contents)
                        publication = soup.find('table', {'class': 'listing'})
                        published_date = publication.find('span').string.split(' ')[-1]
                        published_date = datetime.strptime(published_date, '%Y-%m-%d')
                        if int((start.date() - published_date.date()).days) <= 90:
                            continue
                        if last_message_date.date() <= published_date.date():
                            verification_nag(bug, pkg_name, release)
                        else:
                            for message in bug.messages:
                                m_owner = message.owner
                                message_dc = message.date_created
                                if message_dc == last_message_dc:
                                    if m_owner not in ignored_commenters:
                                        continue
                                    if int((start.date() - message_dc.date()).days) <= 90:
                                        continue
                                    else:
                                        verification_nag(bug, pkg_name, release)
                    except KeyError:
                        logging.info(
                            'SRU bug %d does not exist or is not accessible' % bug.id)


def regression_proposed_finder():
    releases = {}
    for series in ubuntu.series:
        if not series.supported:
            continue
        if series.active:
            releases[series.name] = series

    for release in sorted(releases):
        datereleased = releases[release].datereleased
        for spph in ARCHIVE.getPublishedSources(
                pocket='Proposed', status='Published',
                distro_series=releases[release],
                created_since_date=datereleased):
            package_name = spph.source_package_name
            # for langpack updates, only keep -en as a representative
            # cargo-culted from sru-report
            if (package_name.startswith('language-pack-') and
                package_name not in ('language-pack-en',
                                     'language-pack-en-base')):
                continue
            date_pub = spph.date_published
            version = spph.source_package_version
            change_url = spph.changesFileUrl()

            if not change_url:
                log_print("Package %s in %s has no changes file url" %
                    (package_name, release))
                continue

            package = ubuntu.getSourcePackage(name=package_name)
            tasks = []
            # apport tags bugs package-from-proposed so search for that
            if release != 'lucid':
                for task in package.searchTasks(created_since=date_pub,
                        tags=['package-from-proposed', release],
                        tags_combinator='All'):
                    tasks.append(task)
            # search for bugs reported by apport
            else:
                for tag in APPORT_TAGS:
                    for task in package.searchTasks(
                            tags=[tag, release], created_since=date_pub,
                            tags_combinator='All'):
                        tasks.append(task)
            # also search for ones tagged regression-proposed
            for task in package.searchTasks(
                    tags=['regression-proposed', release],
                    created_since=date_pub, tags_combinator='All'):
                tasks.append(task)

            for task in tasks:
                bug = task.bug
                reg_bug_num = bug.id
                bug_tags = bug.tags
                # ignore bugs that have the tag third-party-packages
                if 'third-party-packages' in bug_tags:
                    log_print('LP: #%s regarding a -proposed package was skipped as it is tagged third-party-packages.' % bug.id)
                    continue
                if 'bot-stop-nagging' in bug_tags:
                    log_print('LP: #%s regarding a -proposed package was skipped as it is tagged bot-stop-nagging.' % bug.id)
                    continue
                if version not in bug.description and \
                    not 'regression-proposed' in bug_tags:
                    # This verbosity seems unnecessary as these are not worth
                    # looking at
                    #log_print('LP: #%s skipped as it does not have the proposed package version.' % bug.id)
                    continue
                sru_bugs = bugs_from_changes(change_url)
                # there are no SRU bugs to comment on
                if len(sru_bugs) == 0:
                    # subscribe myself so I can sort out what to do
                    bug.subscribe(person=BRIAN)
                    continue
                # check to see if any of the sru bugs are already tagged
                # verification-failed
                v_failed = False
                for sru_bug in sru_bugs:
                    sru_bug_tags = sru_bug.tags
                    if 'verification-needed' not in sru_bug_tags:
                        # set v_failed so that it gets skipped later
                        v_failed = True
                        continue
                    if 'verification-failed' in sru_bug_tags or \
                            'verification-failed-%s' % release in sru_bug_tags:
                        log_print('The SRU for package %s already has a '
                              'verification-failed bug in LP: #%s.' %
                              (package_name, sru_bug.id))
                        v_failed = True
                        message_subjects = [m.subject for m in sru_bug.messages]
                        if '[%s/%s] possible regression found' % (package_name, release) in message_subjects:
                            log_print('LP: #%s has already been commented on' % sru_bug.id)
                            break
                        POSSIBLE_REGRESSION_MESSAGE = 'As a part of the Stable Release Updates quality process a search for Launchpad bug reports using the version of %s from %s-proposed was performed and bug %s was found.  Please investigate that bug report to ensure that a regression will not be created by this SRU. In the event that this is not a regression remove the "verification-failed-%s" tag from this bug report and add the tag "bot-stop-nagging" to bug %s (not this bug). Thanks!' % \
                        (package_name, release, reg_bug_num, release, reg_bug_num)
                        sru_bug.subscribe(person=BRIAN)
                        sru_bug.newMessage(content=POSSIBLE_REGRESSION_MESSAGE,
                            subject='[%s/%s] possible regression found' % (package_name, release))
                        break
                if not v_failed:
                    # just use the first SRU bug sru_bugs[0]
                    sru_bug = sru_bugs[0]
                    POSSIBLE_REGRESSION_MESSAGE = 'As a part of the Stable Release Updates quality process a search for Launchpad bug reports using the version of %s from %s-proposed was performed and bug %s was found.  Please investigate this bug report to ensure that a regression will not be created by this SRU. In the event that this is not a regression remove the "verification-failed-%s" tag from this bug report and add the tag "bot-stop-nagging" to bug %s (not this bug). Thanks!' % \
                    (package_name, release, reg_bug_num, release, reg_bug_num)
                    sru_bug.subscribe(person=BRIAN)
                    sru_bug.newMessage(content=POSSIBLE_REGRESSION_MESSAGE,
                        subject='[%s/%s] possible regression found' % (package_name, release))
                    add_tags(sru_bug, ['verification-failed-%s' % release])

def upgrade_failures(since):
    '''Examine upgrade log files for release upgrade calculation failures due
       to xorg-edgers packages, cinnamon ppa, or flightgear being installed'''

    uru = ubuntu.getSourcePackage(name='ubuntu-release-upgrader')
    uru_tasks = uru.searchTasks(order_by='-datecreated',
        tags=['apport-bug', 'dist-upgrade'], tags_combinator='All',
        created_since=since)
    update_manager = ubuntu.getSourcePackage(name='update-manager')
    um_tasks = update_manager.searchTasks(order_by='-datecreated',
        tags=['apport-bug', 'dist-upgrade'], tags_combinator='All',
        created_since=since)
    tasks = [uru_t for uru_t in uru_tasks] + [um_t for um_t in um_tasks]

    COMMENT = "Reviewing the files from your upgrade attempt you seem to \
have enabled a PPA that provides Xorg packages. This will prevent the \
upgrade as that PPA contains different versions of packages essential to \
Ubuntu. The page, https://launchpad.net/~xorg-edgers/+archive/ppa, \
contains details on to how to revert to the official Xorg packages for \
the xorg-edgers PPA.  Instructions for reverting to official Xorg \
packages should be similar for the PPA that you have enabled. \
Thanks.\n\n%s" % AUTO_RESPONSE

    CINNAMON_COMMENT = "You seem to have installed the cinnamon package on \
Ubuntu 13.10. Unfortunately, that package is not available yet for 14.04. \
To upgrade to Ubuntu 14.04 LTS you'll need to either wait for a fix for \
cinnamon to be available or uninstall the package and then upgrade.\n\n%s" \
% AUTO_RESPONSE

    for task in tasks:
        bug = task.bug
        main_log = [a for a in task.bug.attachments if a.title ==
                    'VarLogDistupgradeMainlog.txt']
        if main_log:
            # looking at the first Mainlog attachment
            hosted_file = main_log[0].data
            try:
                hosted_file_buffer = hosted_file.open()
            except OSError as error:
                if error.strerror == 'Connection refused':
                    continue
            contents = hosted_file_buffer.read()
            try:
                torelease = re.search(
                    r"plugins for condition '(\w+)PreCacheOpen' are",
                    contents).groups()[0]
            except AttributeError:
                # AttributeError happens if there is no match
                continue
            if torelease not in RELEASE_TAGS:
                print("LP: #%s has an unknown release" % bug.id)
                continue
            try:
                fromrelease = re.search(
                    r"plugins for condition 'from_(\w+)PreCacheOpen' are",
                    contents).groups()[0]
            except AttributeError:
                # AttributeError happens if there is no match
                continue
            if fromrelease not in RELEASE_TAGS:
                print("LP: #%s has an unknown release" % bug.id)
                continue
            upgrade_tag = '%s2%s' % (fromrelease, torelease)
            add_tags(bug, [upgrade_tag])
        if error_in_attachments(task,
                re.compile(b"(Obsolete|Foreign):.*cinnamon-session"),
                "VarLogDistupgradeMainlog.txt"):
            add_tags(bug, ['cinnamon'])
            bug.newMessage(content=CINNAMON_COMMENT)
            task.status = 'Invalid'
            mark_as_duplicate('1279762', bug, '')
            task.lp_save()
            log_print("LP: #%s dist-upgrade failure due to cinnamon" %
                bug.id)
            continue
        if error_in_attachments(task,
                re.compile(b"Foreign:.*[a-z]+"),
                "VarLogDistupgradeMainlog.txt"):
            add_tags(bug, ['third-party-packages'])
            # log_print("LP: #%s dist-upgrade with Foreign packages" %
            #     bug.id)
        if error_in_attachments(task,
                re.compile(b"Foreign:.*ros-kinetic"),
                "VarLogDistupgradeMainlog.txt"):
            mark_as_duplicate('1611737', bug, '')
            continue
        if error_in_attachments(task,
                re.compile(b"Holding Back flightgear"),
                "VarLogDistupgradeAptlog.txt") and \
                error_in_attachments(task,
                        re.compile(b"Investigating \(9\) flightgear"),
                        "VarLogDistupgradeAptlog.txt"):
            mark_as_duplicate('1308338', bug, '')
            continue
        if not error_in_attachments(task,
                "DEBUG Installing 'xserver-xorg-video-all'",
                "VarLogDistupgradeMainlog.txt"):
            continue
        # https://launchpad.net/~oibaf/+archive/ubuntu/graphics-drivers
        # ~gd~y is a convention for a PPA
        if not error_in_attachments(task,
                re.compile(b"xserver-xorg.*(sarvatt|~gd~[xyz])"),
                "VarLogDistupgradeAptlog.txt"):
            continue
        add_tags(bug, ['ppa', 'xorg-edgers-ppa'])
        bug.newMessage(content=COMMENT)
        task.status = 'Invalid'
        task.lp_save()
        mark_as_duplicate('1069133', bug)
        log_print("LP: #%s dist-upgrade failure due to an Xorg PPA" %
              bug.id)

def nonerased_install_media(since):
    '''Examine installation log file for files indicating that the install
       media, generally a USB stick, was not erased and contains information
       about another distribution on it.'''

    ubiquity = ubuntu.getSourcePackage(name='ubiquity')
    tasks = ubiquity.searchTasks(order_by='-datecreated',
                created_since=since)

    COMMENT = "Reviewing your installation log file it appears that your \
installation media contains some extra data which is causing the \
installation to fail. The standard Ubuntu installation media (and that of \
supported derivatives) does not include this data and will not fail in this \
way, so it must be an issue with the way your installation media was \
prepared.  Subsequently, this bug report is Invalid.  However, to resolve \
the issue should try fully erasing the USB drive which contains the \
installer before recreating it.  Thanks and good luck!\n\n%s" % AUTO_RESPONSE

    for task in tasks:
        bug = task.bug
        if not error_in_attachments(task, re.compile(b"dpkg: error.*console-setup-linux"),
                "UbiquitySyslog.txt"):
            continue
        if not error_in_attachments(task,
                re.compile(b"trying to overwrite.*which is also in package console-setup"),
                "UbiquitySyslog.txt"):
            continue
        if not error_in_attachments(task,
                re.compile(b"deb cdrom:.*(wheezy|jessie|sid)"),
                "UbiquitySyslog.txt"):
            continue
        add_tags(bug, ['console-setup-conflict', 'not-erased-media'])
        bug.newMessage(content=COMMENT)
        task.status = 'Invalid'
        task.lp_save()
        log_print("LP: #%s treated as a install failure due to not-erased media" %
              bug.id)

if __name__ == "__main__":

    logging.basicConfig(filename='crichton-actions.log',
        format='%(asctime)s %(message)s', level=logging.INFO)
    lp = lpl_common.connect(version='devel', bot=True)
    start = datetime.utcnow()
    ubuntu = lp.distributions['ubuntu']
    ARCHIVE = ubuntu.getArchive(name='primary')
    BRIAN = lp.people['brian-murray']
    FBUGS = lp.people['foundations-bugs']

    search_start = start - timedelta(1)
    APPORT_TAGS = (
        'apport-package',
        'apport-bug',
        'apport-crash',
        'apport-kerneloops',
    )
    RELEASE_TAGS = ['lucid', 'maverick', 'natty', 'oneiric', 'precise',
        'quantal', 'raring', 'saucy', 'trusty', 'utopic', 'vivid',
        'wily', 'xenial', 'yakkety', 'zesty', 'artful', 'bionic', 'cosmic',
        'disco', 'eoan', 'focal', 'groovy', 'hirsute', 'impish', 'jammy',
        'kinetic', 'lunar', 'mantic']
    AUTO_RESPONSE = "[This is an automated message.  I apologize if it reached you inappropriately; please just reply to this message indicating so.]"
    print('Started at %s' % start)
    tag_compiz_with_version()
    reassign_initramfs_tools_bugs()
    reassign_no_package_apport_bugs()
    reassign_no_package_apportbug_bugs()
    # 2012-01-06 this and other oneiric ones catch dist upgrade bug reports
    for release in RELEASE_TAGS:
        incomplete_dpkg_buffer_package(release, 'cannot access archive',
            'inaccessible-archive', search_start)
        incomplete_dpkg_buffer_package(release, 'failed to read on buffer copy',
            'failed-to-read', search_start)
        incomplete_dpkg_buffer_package(release, 'failed in write on buffer copy',
            'failed-in-write', search_start)
        incomplete_dpkg_buffer_package(release, 'short read on buffer copy',
            'short-read', search_start)
        incomplete_dpkg_buffer_package(release, 'corrupted filesystem tarfile - corrupted package archive',
            'corrupted-package', search_start)
        incomplete_dpkg_buffer_package(release, 'unexpected end of file or stream',
            'corrupted-package', search_start)
        incomplete_dpkg_buffer_package(release, 'not a debian format archive',
            'not-debian-format', search_start)
        incomplete_dpkg_buffer_package(release, 'dpkg-deb --fsys-tarfile',
            'fsys-tarfile-error', search_start)
        apport_package_update_initramfs(release, search_start)
    apport_package_segfault(search_start)
    ubiquity_syslog_crashes(search_start)
    apport_package_hardware_error(search_start)
    subscribe_teams_to_patched_bugs(search_start)
    package_manager_triaging(search_start)
    no_package_generic_comment(search_start)
    upgrade_failures(search_start)
    nonerased_install_media(search_start)
    # the following 2 are disabled on cranberry and need investigation
    #sru_verification_nag()
    #regression_proposed_finder()
    stop = datetime.utcnow()
    print('Stopped at %s' % stop)

    # disabled jobs
    # no good resolution for this yet
    # apport_package_debconf_passthrough_errors((start - timedelta(100)))
    # false positives per slangasek
    # lzma_encoder_error(search_start)
    # 2011-05-03 - linux package hook no longer asks about regressions
    # retag_linux_regression_updates()
    # 2023-08-16 might be interesting for historians
    # corrupt_pyc_crashes(search_start)
    # apport_package_dupe_sig_fixer(search_start)

    #retag_regression_potential()
    # 2011-03-02 - being run by didrocks
    #unity_packages = ['bamf', 'nux', 'dee', 'unity-place-files',
    #                  'unity-place-applications', 'libunity', 'unity']
    #for package in unity_packages:
    #    open_upstream_project_task(package, 'unity', search_start)