~laney/ubuntu-archive-tools/retry-autopkgtest-regressions-bileto-v2

#!/usr/bin/python

# Copyright (C) 2009, 2010, 2011, 2012  Canonical Ltd.
# Copyright (C) 2010  Scott Kitterman <scott@kitterman.com>
# Author: Martin Pitt <martin.pitt@canonical.com>
# Author: Brian Murray <brian.murray@canonical.com>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

'''Show and approve changes in an unapproved upload.

Generate a debdiff between current source package in a given release and the
version in the unapproved queue, and ask whether or not to approve the upload.
Approve upload and then comment on the SRU bugs regarding verification process.

USAGE:
    sru-review -b -s precise isc-dhcp
'''

from __future__ import print_function

import gzip
import optparse
import os
import re
import subprocess
import sys
import tempfile
try:
    from urllib.parse import quote
    from urllib.request import urlopen, urlretrieve
except ImportError:
    from urllib import quote, urlopen, urlretrieve
import webbrowser

from launchpadlib.launchpad import Launchpad


def parse_options():
    '''Parse command line arguments.

    Return (options, source_package) tuple.
    '''
    parser = optparse.OptionParser(
        usage='Usage: %prog [options] source_package')
    parser.add_option(
        "-l", "--launchpad", dest="launchpad_instance", default="production")
    parser.add_option(
        "-s", dest="release", default=default_release, metavar="RELEASE",
        help="release (default: %s)" % default_release)
    parser.add_option(
        "-p", dest="ppa", metavar="LP_USER/PPA_NAME",
        help="Check a PPA instead of the Ubuntu unapproved queue")
    parser.add_option(
        "-b", "--browser", dest="browser", action="store_true",
        default=True, help="Open Launchpad bugs in browser")
    parser.add_option(
        "-v", "--view", dest="view", action="store_true",
        default=True, help="View debdiff in pager")
    parser.add_option(
        "--no-diff", dest="diff", action="store_false", default=True,
        help=(
            "Don't fetch debdiff, assuming that it has been reviewed "
            "separately (useful for copies)"))
    parser.add_option(
        "-q", "--queue", dest='queue',
        help='Use a specific queue instead of Unapproved',
        default="Unapproved",
        choices=("Unapproved", "New", "Rejected"),
        metavar='QUEUE')

    (opts, args) = parser.parse_args()

    if len(args) != 1:
        parser.error('Need to specify one source package name')

    return (opts, args[0])


def parse_changes(changes_url):
    '''Parse .changes file.

    Return dictionary with interesting information: 'bugs' (list),
    'distribution'.
    '''
    info = {'bugs': []}
    for line in urlopen(changes_url):
        if line.startswith('Distribution:'):
            info['distribution'] = line.split()[1]
        if line.startswith('Launchpad-Bugs-Fixed:'):
            info['bugs'] = sorted(set(line.split()[1:]))
        if line.startswith('Version:'):
            info['version'] = line.split()[1]
    return info


def from_queue(options, archive, sourcepkg, series):
    '''Get package_upload from LP and debdiff from queue page.

    Return (package_upload, changes URL, debdiff URL) tuple.
    '''
    queues = {'New': 0, 'Unapproved': 1, 'Rejected': 4}
    queue_url = ('https://launchpad.net/ubuntu/%s/+queue?'
                 'queue_state=%s&batch=300' %
                 (series.name, queues[options.queue]))
    uploads = [upload for upload in
               series.getPackageUploads(archive=archive, exact_match=True,
                                        name=sourcepkg, pocket='Proposed',
                                        status=options.queue)]
    if len(uploads) == 0:
        print('ERROR: Queue does not have an upload of this source.',
              file=sys.stderr)
        sys.exit(1)
    if len(uploads) > 1:
        print('ERROR: Queue has more than one upload of this source, '
              'please handle manually', file=sys.stderr)
        sys.exit(1)
    upload = uploads[0]

    if upload.contains_copy:
        archive = upload.copy_source_archive
        pubs = archive.getPublishedSources(
            exact_match=True, source_name=upload.package_name,
            version=upload.package_version)
        if pubs:
            changes_url = pubs[0].changesFileUrl()
        else:
            print("ERROR: Can't find source package %s %s in %s" %
                  (upload.package_name, upload.package_version,
                   archive.web_link),
                  file=sys.stderr)
            sys.exit(1)
    else:
        changes_url = upload.changes_file_url

    if options.diff:
        oops_re = re.compile('class="oopsid">(OOPS[a-zA-Z0-9-]+)<')
        debdiff_re = re.compile(
            'href="(http://launchpadlibrarian.net/'
            '\d+/%s_[^"_]+_[^_"]+\.diff\.gz)">\s*diff from' %
            re.escape(quote(sourcepkg)))

        queue_html = urlopen(queue_url).read()

        m = oops_re.search(queue_html)
        if m:
            print('ERROR: Launchpad failure:', m.group(1), file=sys.stderr)
            sys.exit(1)

        m = debdiff_re.search(queue_html)
        if not m:
            print('ERROR: queue does not have a debdiff', file=sys.stderr)
            sys.exit(1)
        debdiff_url = m.group(1)
        #print('debdiff URL:', debdiff_url, file=sys.stderr)
    else:
        debdiff_url = None

    return (upload, changes_url, debdiff_url)


def from_ppa(options, sourcepkg, user, ppaname):
    '''Get .changes and debdiff from a PPA.

    Return (changes URL, debdiff URL) pair.
    '''
    changes_re = re.compile(
        'href="(https://launchpad.net/[^ "]+/%s_[^"]+_source.changes)"' %
        re.escape(quote(sourcepkg)))
    sourcepub_re = re.compile(
        'href="(\+sourcepub/\d+/\+listing-archive-extra)"')
    #debdiff_re = re.compile(
    #    'href="(https://launchpad.net/.[^ "]+.diff.gz)">diff from')

    changes_url = None
    changes_sourcepub = None
    last_sourcepub = None

    for line in urlopen(ppa_url % (user, ppaname, options.release)):
        m = sourcepub_re.search(line)
        if m:
            last_sourcepub = m.group(1)
            continue
        m = changes_re.search(line)
        if m:
            # ensure that there's only one upload
            if changes_url:
                print('ERROR: PPA has more than one upload of this source, '
                      'please handle manually', file=sys.stderr)
                sys.exit(1)
            changes_url = m.group(1)
            assert changes_sourcepub is None, (
                'got two sourcepubs before .changes')
            changes_sourcepub = last_sourcepub

    #print('changes URL:', changes_url, file=sys.stderr)

    # the code below works, but the debdiffs generated by Launchpad are rather
    # useless, as they are against the final version, not what is in
    # -updates/-security; so disable

    #if options.diff:
    #    # now open the sourcepub and get the URL for the debdiff
    #    changes_sourcepub = changes_url.rsplit('+', 1)[0] + changes_sourcepub
    #    #print('sourcepub URL:', changes_sourcepub, file=sys.stderr)
    #    sourcepub_html = urlopen(changes_sourcepub).read()

    #    m = debdiff_re.search(sourcepub_html)
    #    if not m:
    #        print('ERROR: PPA does not have a debdiff', file=sys.stderr)
    #        sys.exit(1)
    #    debdiff_url = m.group(1)
    #    #print('debdiff URL:', debdiff_url, file=sys.stderr)
    #else:
    debdiff_url = None

    return (changes_url, debdiff_url)


def process_bug(launchpad, upload, num):
    bug_target_re = re.compile(
        r'/ubuntu/(?:(?P<suite>[^/]+)/)?\+source/(?P<source>[^/]+)$')
    bug = launchpad.bugs[num]
    sourcepkg = upload.package_name
    release = upload.distroseries.name
    for task in bug.bug_tasks:
        # Ugly; we have to do URL-parsing to figure this out.
        # /ubuntu/+source/foo can be fed to launchpad.load() to get a
        # distribution_source_package, but /ubuntu/hardy/+source/foo can't.
        match = bug_target_re.search(task.target.self_link)
        if (not match or
            (sourcepkg and
             match.group('source') != sourcepkg)):
            print("Ignoring task %s in bug %s" % (task.web_link, num))
            continue
        if (match.group('suite') == upload.distroseries.name and
            task.status not in ("Invalid", "Won't Fix", "Fix Committed",
                                "Fix Released")):
            task.status = "Fix Committed"
            task.lp_save()
            print("Success: task %s in bug %s" % (task.web_link, num))

    bug.subscribe(person=launchpad.people['ubuntu-sru'])
    bug.subscribe(person=launchpad.people['sru-verification'])

    if not sourcepkg or 'linux' not in sourcepkg:
        btags = bug.tags
        for t in ('verification-failed', 'verification-done'):
            if t in btags:
                # this dance is needed due to
                # https://bugs.launchpad.net/launchpadlib/+bug/254901
                tags = btags
                tags.remove(t)
                bug.tags = tags
        bug.lp_save()

        if 'verification-needed' not in btags:
            # this dance is needed due to
            # https://bugs.launchpad.net/launchpadlib/+bug/254901
            tags = btags
            tags.append('verification-needed')
            bug.tags = tags
            bug.lp_save()

    text = ('Hello %s, or anyone else affected,\n\n' %
            re.split(r'[,\s]', bug.owner.display_name)[0])

    if sourcepkg:
        text += 'Accepted %s into ' % sourcepkg
    else:
        text += 'Accepted into '
    if sourcepkg and release:
        text += ('%s-proposed. The package will build now and be available at '
                 'http://launchpad.net/ubuntu/+source/%s/%s in a few hours, '
                 'and then in the -proposed repository.\n\n' % (
                     release, sourcepkg, upload.package_version))
    else:
        text += ('%s-proposed. The package will build now and be available in '
                 'a few hours in the -proposed repository.\n\n' % (
                     release))

    text += ('Please help us by testing this new package.  See '
             'https://wiki.ubuntu.com/Testing/EnableProposed for '
             'documentation how to enable and use -proposed.  Your feedback '
             'will aid us getting this update out to other Ubuntu users.\n\n'
             'If this package fixes the bug for you, please add a comment to '
             'this bug, mentioning the version of the package you tested, and '
             'change the tag from verification-needed to verification-done. '
             'If it does not fix the bug for you, please add a comment '
             'stating that, and change the tag to verification-failed.  In '
             'either case, details of your testing will help us make a better '
             'decision.\n\n'
             'Further information regarding the verification process can be '
             'found at '
             'https://wiki.ubuntu.com/QATeam/PerformingSRUVerification .  '
             'Thank you in advance!')
    bug.newMessage(content=text, subject='Please test proposed package')


if __name__ == '__main__':

    default_release = 'trusty'
    ppa_url = ('https://launchpad.net/~%s/+archive/%s/+packages?'
               'field.series_filter=%s')

    (opts, sourcepkg) = parse_options()

    launchpad = Launchpad.login_with('sru-review', opts.launchpad_instance,
                                     version="devel")
    ubuntu = launchpad.distributions['ubuntu']
    series = ubuntu.getSeries(name_or_version=opts.release)
    archive = ubuntu.main_archive

    if opts.ppa:
        (user, ppaname) = opts.ppa.split('/', 1)
        (changes_url, debdiff_url) = from_ppa(opts, sourcepkg, user, ppaname)
    else:
        (upload, changes_url, debdiff_url) = from_queue(
            opts, archive, sourcepkg, series)

    # Check for existing version in proposed
    if series != ubuntu.current_series:
        existing = [
            pkg for pkg in archive.getPublishedSources(
                exact_match=True, distro_series=series, pocket='Proposed',
                source_name=sourcepkg, status='Published')]
        updates = [
            pkg for pkg in archive.getPublishedSources(
                exact_match=True, distro_series=series, pocket='Updates',
                source_name=sourcepkg, status='Published')]
        for pkg in existing:
            if pkg not in updates:
                changesfile_url = pkg.changesFileUrl()
                changes = parse_changes(changesfile_url)
                msg = ('''\
*******************************************************
*
* WARNING: %s already published in Proposed (%s)
* SRU Bug: LP: #%s
*
*******************************************************''' %
                       (sourcepkg, pkg.source_package_version,
                        ' LP: #'.join(changes['bugs'])))
                print(msg, file=sys.stderr)
                print('''View the debdiff anyway? [yN]''', end="")
                response = sys.stdin.readline()
                if response.strip().lower().startswith('y'):
                    continue
                else:
                    print('''Exiting''')
                    sys.exit(1)

    debdiff = None
    if debdiff_url:
        debdiff = gzip.open(urlretrieve(debdiff_url)[0]).read()
    elif opts.diff:
        print('No debdiff available')

    # parse changes and open bugs first since we are using subprocess
    # to view the diff
    changes = parse_changes(changes_url)

    if opts.browser and changes['bugs']:
        for b in changes['bugs']:
            # use a full url so the right task is highlighted
            webbrowser.open('https://bugs.launchpad.net/ubuntu/+source/'
                            '%s/+bug/%s' % (upload.package_name, b))

    if debdiff and opts.view:
        tfile = tempfile.mkstemp()
        os.write(tfile[0], debdiff)
        os.close(tfile[0])
        ret = subprocess.call(["sensible-pager", tfile[1]])
        os.remove(tfile[1])

    if opts.ppa:
        print('\nTo copy from PPA to distribution, run:\n'
              '  copy-package -b --ppa=%s --ppa-name=%s -s %s --to-primary '
              '--to-suite %s-proposed -y %s\n' %
              (user, ppaname, opts.release, opts.release, sourcepkg),
              file=sys.stderr)
        sys.exit(0)

    print("Accept the package into -proposed? [yN] ", end="")
    response = sys.stdin.readline()
    if response.strip().lower().startswith('y'):
        upload.acceptFromQueue()
        print("Accepted")
        if changes['bugs']:
            for bug_num in changes['bugs']:
                process_bug(launchpad, upload, bug_num)
    else:
        print("REJECT the package from -proposed? [yN] ", end="")
        response = sys.stdin.readline()
        if response.strip().lower().startswith('y'):
            print("Please give a reason for the rejection.")
            reason = sys.stdin.readline().strip()
            if reason == '':
                print("A reason must be provided.")
                sys.exit(1)
            upload.rejectFromQueue(comment=reason)
            print("Rejected")
        else:
            print("Not accepted")
            sys.exit(1)