~ursinha/ubuntu-ci-services-itself/401-copying-di-check

#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright (C) 2014 Canonical
#
# Authors:
#  Didier Roche
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; version 3.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

import argparse
import json
import logging
import os
import subprocess
import sys
import textwrap
import time

from cupstream2distro import (launchpadmanager, packageinppamanager)
from cupstream2distro.packageinppa import PackageInPPA
from cupstream2distro.packagemanager import list_packages_info_in_str
from cupstream2distro.settings import (
    TIME_BETWEEN_PPA_CHECKS, TIME_BEFORE_STOP_LOOKING_FOR_SOURCE_PUBLISH)

sys.path.append(os.path.join(os.path.dirname(__file__), '../ci-utils'))
from ci_utils import dump_stack
from ci_utils.ticket_states import (
    SubTicketWorkflowStep,
    SubTicketWorkflowStepStatus
)

EXISTING_SOURCE_MESSAGE = textwrap.dedent('''\
        ERROR: A source package with an equal or higher version
        of {} exists in {}.
        The upload of {} with version {} is failed.
        Source package uploads must have a higher version.''')
FAILED_SOURCE_MESSAGE = textwrap.dedent('''\
        The source package build has failed in {}.
        Please review the build logs for the failed package(s).''')
MISSING_SOURCE_MESSAGE = textwrap.dedent('''\
        The source package for {} with version {}
        failed to upload to {}.
        Please check that the source package is signed with a signature of
        a launchpad user with permission to upload to the PPA.
        Also check for a rejection message sent to the package signer.''')
PASSED_SOURCE_MESSAGE = textwrap.dedent('''\
        The source packages have been built in {}.''')


def parse_arguments():
    parser = argparse.ArgumentParser(
        description="Watch for published package in a ppa",
        epilog="""series and ppa options can be set by the corresponding long
        option name env variables as well""")
    parser.add_argument("-u", "--upload-list",
                        help="""A json formated list of packages to check.
                        The format is [{"name": package_name,
                        "version": package_version, "architecture": arch}]""")
    parser.add_argument("-s", "--series",
                        help="Serie used to build the package")
    parser.add_argument("-p", "--ppa",
                        help="""PPA to publish this package to (for instance:
                        'ubuntu-unity/daily-build')""")
    parser.add_argument("-a", "--arch",
                        default=None,
                        help="Only consider the provided target")
    parser.add_argument("-d", "--destppa",
                        help="""Consider this destppa instead of
                        {series}-proposed""")
    return parser.parse_args()


def get_environment_variables(args):
    series = args.series
    ppa = args.ppa
    if not series:
        series = os.getenv("series")
    if not ppa:
        ppa = os.getenv("ppa")
    return (series, ppa)


def get_ppa(ppa):
    return launchpadmanager.get_ppa(ppa)


def get_launchpad_log(url):
    '''Retrieves the build log from launchpad (thanks to celso).'''
    if not url:
        return None
    api_url = url.replace('/launchpad.net/', '/api.launchpad.net/devel/')
    lp = launchpadmanager.get_launchpad()
    return lp._browser.get(api_url)


def get_build_logs(datastore, ppa, package):
    '''Constructs artifacts from the build logs for the given package.'''
    source_name = package.source_name
    version = package.version

    collection = ppa.getPublishedSources(source_name=source_name,
                                         version=version)
    builds = collection[0].getBuilds()

    artifacts = []
    for build in builds:
        build_log = get_launchpad_log(build.build_log_url)
        upload_log = get_launchpad_log(build.upload_log_url)
        if build_log:
            artifact_name = 'package_build.{}.{}.build.log'.format(
                source_name, build.arch_tag)
            location = datastore.put_file(artifact_name, build_log,
                                          'text/plain')
            artifacts.append({
                'name': artifact_name,
                'reference': location,
                'type': 'LOGS'})
        if upload_log:
            artifact_name = 'package_build.{}.{}.upload.log'.format(
                source_name, build.arch_tag)
            location = datastore.put_file(artifact_name, upload_log,
                                          'text/plain')
            artifacts.append({
                'name': artifact_name,
                'reference': location,
                'type': 'LOGS'})
    return artifacts


def create_package_status(package, step, status, message=None, artifacts=None):
    return {
        'name': package.source_name,
        'version': package.version,
        'step': step.value,
        'step_text': step.title,
        'status': status.value,
        'status_text': status.title,
        'message': message,
        'artifacts': artifacts,
    }


def collect_subticket_status(datastore, ppa, packages_not_in_ppa,
                             packages_building, packages_failed,
                             packages_complete):
    '''Collects the sets of package build status into a subticket_status.'''
    subticket_status = []
    for package in packages_not_in_ppa:
        subticket_status.append(create_package_status(
            package, SubTicketWorkflowStep.QUEUED,
            SubTicketWorkflowStepStatus.PKG_BUILDING_WAITING))

    for package in packages_building:
        subticket_status.append(create_package_status(
            package, SubTicketWorkflowStep.PKG_BUILDING,
            SubTicketWorkflowStepStatus.PKG_BUILDING_INPROGRESS))

    for package in packages_failed:
        subticket_status.append(create_package_status(
            package, SubTicketWorkflowStep.COMPLETED,
            SubTicketWorkflowStepStatus.PKG_BUILDING_FAILED,
            FAILED_SOURCE_MESSAGE.format(ppa.web_link),
            get_build_logs(datastore, ppa, package)))

    for package in packages_complete:
        subticket_status.append(create_package_status(
            package, SubTicketWorkflowStep.COMPLETED,
            SubTicketWorkflowStepStatus.PKG_BUILDING_COMPLETED,
            PASSED_SOURCE_MESSAGE.format(ppa.web_link),
            get_build_logs(datastore, ppa, package)))

    return subticket_status


def get_versions_for_source_package(series, ppa, source_name):
    try:
        source = ppa.getPublishedSources(exact_match=True,
                                         source_name=source_name,
                                         distro_series=series)[0]
        return source.source_package_version
    except (KeyError, IndexError):
        return None


def check_ppa(series, ppa, dest_ppa, arch, upload_list):
    # Prepare launchpad connection:
    lp_series = launchpadmanager.get_series(series)
    monitored_ppa = launchpadmanager.get_ppa(ppa)
    if dest_ppa:
        dest_archive = get_ppa(dest_ppa)
    else:
        dest_archive = launchpadmanager.get_ubuntu_archive()
    logging.info('Series: {}'.format(lp_series))
    logging.info('Monitoring PPA: {}'.format(monitored_ppa))
    logging.info('Destination Archive: {}'.format(dest_archive))

    failed = False
    check_status = []
    for source_package in upload_list:
        source = source_package['name']
        version = source_package['version']
        logging.info('Inspecting upload: {} - {}'.format(source, version))
        message_list = []
        subticket_status = {
            'name': source,
            'id': source_package['id'],
            'version': version,
        }
        wf_step = SubTicketWorkflowStep.NEW
        wf_status = SubTicketWorkflowStepStatus.NEW
        check_status.append(subticket_status)
        for ppa in [monitored_ppa, dest_archive]:
            last_version = get_versions_for_source_package(lp_series, ppa,
                                                           source)
            logging.info('Last source: {}'.format(last_version))

            if last_version:
                try:
                    subprocess.check_call(['dpkg', '--compare-versions',
                                           last_version, 'lt', version])
                except subprocess.CalledProcessError:
                    # The version in the PPA is equal or higher then the
                    # source package, the build will fail
                    wf_step = SubTicketWorkflowStep.COMPLETED
                    wf_status = SubTicketWorkflowStepStatus.PKG_BUILDING_FAILED
                    message_list.append(EXISTING_SOURCE_MESSAGE.format(
                        last_version, ppa.web_link, source, version))
                    failed = True
        subticket_status['step'] = wf_step.value
        subticket_status['step_text'] = wf_step.title
        subticket_status['status'] = wf_status.value
        subticket_status['status_text'] = wf_status.title
        subticket_status['message'] = '\n'.join(message_list)

    return (failed, check_status)


def find_subticket(upload_list, name, version):
    for subticket in upload_list:
        if subticket['name'] == name and subticket['version'] == version:
            return subticket
    return None


def watch_ppa(datastore, time_start, series, ppa, dest_ppa, arch, upload_list):
    # Prepare launchpad connection:
    lp_series = launchpadmanager.get_series(series)
    monitored_ppa = launchpadmanager.get_ppa(ppa)
    if dest_ppa:
        dest_archive = get_ppa(dest_ppa)
    else:
        dest_archive = launchpadmanager.get_ubuntu_archive()
    logging.info('Series: {}'.format(lp_series))
    logging.info('Monitoring PPA: {}'.format(monitored_ppa))
    logging.info('Destination Archive: {}'.format(dest_archive))

    # Get archs available and archs to ignore
    (available_archs_in_ppa,
     arch_all_arch) = launchpadmanager.get_available_and_all_archs(
        lp_series, monitored_ppa)
    (archs_to_eventually_ignore,
     archs_to_unconditionally_ignore) = launchpadmanager.get_ignored_archs()
    logging.info('Arches available in ppa: {}'.format(available_archs_in_ppa))
    logging.info('All arch in ppa: {}'.format(arch_all_arch))
    logging.info('Arches to eventually ignore: {}'.format(
        archs_to_eventually_ignore))
    logging.info('Arches to unconditionally ignore: {}'.format(
        archs_to_unconditionally_ignore))

    # Collecting all packages that have been uploaded to the ppa
    packages_not_in_ppa = set()
    packages_building = set()
    packages_failed = set()
    packages_complete = set()
    for source_package in upload_list:
        source = source_package['name']
        version = source_package['version']
        archs = source_package['architecture']
        logging.info('Inspecting upload: {} - {}'.format(source, version))
        packages_not_in_ppa.add(PackageInPPA(source, version, monitored_ppa,
                                             dest_archive, lp_series,
                                             available_archs_in_ppa,
                                             arch_all_arch,
                                             archs_to_eventually_ignore,
                                             archs_to_unconditionally_ignore,
                                             package_archs=archs))

    # packages_not_in_ppa are packages that were uploaded and are expeceted
    # to eventually appear in the ppa.
    logging.info('Packages not in PPA: {}'.format(
        list_packages_info_in_str(packages_not_in_ppa)))

    # Check the status regularly on all packages
    # TODO The following is the original check loop. This can be extracted
    #    and optimized.
    logging.info("Checking the status for {}".format(
        list_packages_info_in_str(
            packages_not_in_ppa.union(packages_building))))
    packageinppamanager.update_all_packages_status(
        packages_not_in_ppa, packages_building, packages_failed,
        packages_complete, arch)

    status = collect_subticket_status(datastore, monitored_ppa,
                                      packages_not_in_ppa, packages_building,
                                      packages_failed, packages_complete)
    for package in status:
        subticket = find_subticket(upload_list, package['name'],
                                   package['version'])
        if not subticket:
            # No match, this should not happen
            return (1, status)

        # Populate the remaining fields necessary to update ticket status
        package['id'] = subticket['id']
        package['resource'] = subticket['resource']

    logging.info("Status: {}".format(status))
    # if we have no package building or failing and have wait for
    # long enough to have some package appearing in the ppa, exit
    if (packages_not_in_ppa and not packages_building and
        ((time.time() - time_start) >
         TIME_BEFORE_STOP_LOOKING_FOR_SOURCE_PUBLISH)):
        # TODO return error on the missing packages
        logging.info(
            "Some source packages were never published in the ppa: "
            "{}".format(list_packages_info_in_str(packages_not_in_ppa)))
        for package in status:
            # If any packages are WAITING, set them to FAILED
            if package['status'] == SubTicketWorkflowStepStatus.WAITING.value:
                package['step'] = SubTicketWorkflowStep.COMPLETED.value,
                package['status'] = \
                    SubTicketWorkflowStepStatus.PKG_BUILDING_COMPLETED.value
            # Add a message indicating the cause of the failure
            message_list.append(MISSING_SOURCE_MESSAGE.format(
                package['name'], package['version'], monitored_ppa.web_link))
        status['message'] = '\n'.join(message_list)
        return (1, status)

    # break out of status check loop if all packages have arrived in
    # the ppa and have completed building
    if not packages_not_in_ppa and not packages_building:
        if packages_failed:
            # TODO Return package failure info
            logging.info(
                "Some of the packages failed to build: {}".format(
                    list_packages_info_in_str(packages_failed)))
            return (1, status)
        return (0, status)

    # -1 indicates to retry
    # TODO return useful status about what is still in progress
    return (-1, status)


def main():
    '''Provides usage through the command line.'''
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s "
                        "%(message)s")

    args = parse_arguments()
    (series, ppa) = get_environment_variables(args)
    # upload_list is a json formated text field which contains:
    #   [{"name": source_package_name,
    #     "version": source_package_version,
    #     "architecture": architecture_to_monitor}]
    # Example:
    #   [{"name": "cupstream2distro-config",
    #     "version": "0.3.1",
    #     "architecture": "all"}]
    upload_list = json.loads(args.upload_list)

    if not series or not ppa:
        logging.error("Missing compulsory environment variables (ppa, series) "
                      "watching: {}, series: {}".format(ppa, series))
        return 1

    time_start = time.time()
    while True:
        (ret, status) = watch_ppa(time_start, series, ppa,
                                  args.destppa, args.arch, upload_list)
        if ret == -1:
            time.sleep(TIME_BETWEEN_PPA_CHECKS)
        else:
            return ret


if __name__ == '__main__':
    dump_stack.install_stack_dump_signal()
    sys.exit(main())