~ubuntu-on-ec2/vmbuilder/jenkins_kvm-add-azure-cc

#!/usr/bin/python3
#
# Copyright 2014, Ben Howard <ben.howard@canonical.com>
# Copyright 2014, Canonical Group, Ltd.
#
# Evaluate a tab/space deliminated manifest in "<pkg_nmae> <ver>" format
# against what is in the archive.


import argparse
import bz2
import logging
import os
import re
import subprocess
import sys
import time

# Use our version of pylib
mypath = os.path.dirname(os.path.realpath(__file__))
pylib_path = "%s/pylib" %  mypath
sys.path[0] = pylib_path
import requests

if 2 == sys.version_info.major:
    raise Exception("I am incompatiable with Python2!")

build_arches = {
    "default": ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el'],
    "override": None,
    "precise": ['amd64', 'i386', 'armhf'],
    "lucid": ['amd64', 'i386'],
}

host_override = {
    'default': "archive.ubuntu.com",
    'arm64': "ports.ubuntu.com",
    'armhf': "ports.ubuntu.com",
    'ppc64el': "ports.ubuntu.com",
}

DAILY_PATTERN = "http://cloud-images.ubuntu.com/%(SUITE)s/current/%(SUITE)s-server-cloudimg-%(ARCH)s.manifest"  # noqa
RELEASE_PATTERN = "http://cloud-images.ubuntu.com/releases/%(SUITE)s/release/ubuntu-%(VERSION)s-server-cloudimg-%(ARCH)s.manifest"  # noqa

# Setup the logger
logger = logging.getLogger('_archivewatch_')
logging.basicConfig(format=
                    '%(asctime)s  %(levelname)s - %(message)s')
logger.setLevel(logging.DEBUG)


def dpkg_version_greater_than(a, b):
    """
    Use dpkg to check if a > b
    """
    return_code = subprocess.call(
        ['dpkg', '--compare-versions', a, 'gt', b])
    return return_code == 0


def get_suite_version(suite):
    with open("/usr/share/distro-info/ubuntu.csv") as f:
        for line in f.readlines():
            version, _, codename = line.split(",")[:3]
            if codename == suite:
                return version.split()[0]
    return None


def suite_archs(suite):
    if build_arches["override"] is not None:
        return build_arches["override"]
    elif suite not in build_arches:
        return build_arches['default']
    return build_arches[suite]


def iter_archs(suite):
    for archs in suite_archs(suite):
        yield archs


def get_archive_addresses(archive, suite, pocket, arch, ppas=None,
                          proposed=False):
    """
    Iterator for returning the archive addresses
    """

    # Switch the default archive for ports
    if arch in host_override and archive in host_override['default']:
        archive = host_override[arch]

    # port uses a different URL pattern
    if "ports" not in archive:
        archive = "%s/ubuntu" % archive

    logger.info("Using %s as archive host for %s" % (archive, arch))
    base_string = "http://%s/dists/%s/%s/binary-%s/Packages.bz2"
    yield base_string % (archive, suite, pocket, arch)

    # Handle Security updates
    if archive.split('/')[0] in host_override['default']:
        yield base_string % (archive.replace("archive", "security"), suite,
                             pocket, arch)
        yield base_string % (archive, "%s-security" % suite, pocket, arch)

        if proposed:
            yield base_string % (archive, "%s-proposed" % suite, pocket, arch)

    elif archive == "ports.ubuntu.com":
        yield base_string % (archive, "%s-security" % suite, pocket, arch)

        if proposed:
            yield base_string % (archive, "%s-proposed" % suite, pocket, arch)

    yield base_string % (archive, "%s-updates" % suite, pocket, arch)

    def ppa_to_address(ppa):
        if 'ppa:' in ppa:
            ppa = ppa.replace("ppa:", "")
            ppa_base = "http://ppa.launchpad.net/%s/ubuntu/dists/%s/%s/binary-%s/Packages.bz2"  # noqa
            return ppa_base % (ppa, suite, pocket, arch)
        else:
            return ppa

    # allow us to play against PPA's too
    if isinstance(ppas, list):
        for ppa in ppas:
            yield ppa_to_address(ppa)
    elif ppas:
        yield ppa_to_address(ppas)


def get_manifest(manifest_loc):
    """
    Read the manifest in and return it as a dict

    If the manifest is absent in a way that suggests we should be
    building this arch, None will be returned.
    """
    logger.info("Reading manifest %s" % manifest_loc)
    pkg_list = {}
    raw_list = None

    if "http" in manifest_loc:
        logger.info("     Requesting manifest over http")
        r = requests.get(manifest_loc)
        if r.status_code != requests.codes.ok:
            logger.debug("     Request failed; body:\n%s", r.content)
            if r.status_code == requests.codes.not_found:
                return None
            r.raise_for_status()
        raw_list = bytes.decode(r.content)

    else:
        if not os.path.exists(manifest_loc):
            return None
        with open(manifest_loc, 'r') as f:
            raw_list = f.read()

    for line in raw_list.splitlines():
        try:
            pkg, ver = line.split()
        except ValueError:
            logger.exception('     Problem splitting line; full manifest:\n%s',
                             raw_list)
            raise
        pkg_list[pkg] = ver

    logger.info("     found %s packages" % len(pkg_list))
    return pkg_list


def get_archive_meta(archive, suite, pocket, arch, ppas=None, proposed=False):
    """
    Get the archive meta data and decompress it
    """
    archive_content = {}
    pkg_re = re.compile("^Package.*")
    ver_re = re.compile("^Version.*")

    for url in get_archive_addresses(archive, suite, pocket, arch, ppas,
                                     proposed):
        logger.info("Fetching archive %s" % url)
        r = requests.get(url)

        logger.info("     Evaluating packages....")
        count = 0
        pkg_name = None
        pkg_ver = None

        content = bz2.decompress(r.content)
        for line in content.splitlines():
            try:
                line = line.decode()
            except Exception as e:
                logger.info("Can't decode: %s" % line)
                logger.warn(e)

            if not pkg_name and pkg_re.match(line):
                pkg_name = line.split(":", 1)[-1].strip()
                pkg_name = pkg_name.replace(":%s" % arch, "")
            elif pkg_name and ver_re.match(line):
                pkg_ver = line.split(":", 1)[-1].strip()

            if pkg_name and pkg_ver:
                count += 1
                found_ver = archive_content.get(pkg_name)
                if not found_ver or dpkg_version_greater_than(pkg_ver,
                                                              found_ver):
                    archive_content[pkg_name] = pkg_ver

                (pkg_name, pkg_ver) = (None, None)


        logger.info("     processed %s packages" % count)

    logger.info("Considering %s packages in archives" % len(archive_content))
    return archive_content


def check_pkg_versions(pkgs, wait_pkgs):
    """
    Check if package versions requests in wait_pkgs exist in metadata. Expects
    that 'wait_pkgs' be formated as '<pkg>::<version>', i.e. udev::1.0.1
    """
    not_met = []
    if not isinstance(wait_pkgs, list):
        wait_pkgs = [wait_pkgs]

    for wpkg in wait_pkgs:
        (wpkg_name, wpkg_ver) = wpkg.split("::", 1)
        logger.info("Looking for package %s %s" % (wpkg_name, wpkg_ver))

        for arch in pkgs:
            apkgs = pkgs.get(arch)
            arch_v = apkgs.get('archive')
            manfst_v = apkgs.get('installed')

            in_archive = arch_v.get(wpkg_name, None)
            in_image = manfst_v.get(wpkg_name, None)

            logger.info("Package %s:%s - MAN (%s) ARCHIVE (%s)" % (
                        wpkg_name, arch, in_image, in_archive))

            if in_archive != wpkg_ver and in_image != wpkg_ver:
                logger.info("   requirement is not meet")
                not_met.append(wpkg)

    if len(not_met) > 0:
        return (False, not_met)
    return True, []


def compare_meta(pkgs):
    """
    Compare the packages to see if things are different
    """
    diffs = {}
    should_build = []
    logger.info("Processing deltas....this may take a few minutes.")
    for arch in pkgs:
        logger.info("Looking at {}".format(arch))
        diffs[arch] = {}
        apkgs = pkgs.get(arch)
        if apkgs.get('new_build', None) is not None:
            logger.info(
                "{} hasn't been built before; skipping delta checks".format(
                    arch))
            should_build.append(arch)
            continue
        arch_v = apkgs.get('archive')
        manfst_v = apkgs.get('installed')

        diff_msgs = []

        for ipkg, iver in manfst_v.items():
            ipkg = ipkg.split(':')[0]
            aver = arch_v.get(ipkg, None)

            if not aver or aver in iver:
                continue

            if dpkg_version_greater_than(aver, iver):
                diff_msgs.append("{}:{} (MAN: {}) (ARCHIVE: {})".format(
                                 ipkg, arch, iver, aver))
                diffs[arch][ipkg] = (manfst_v, arch_v)
                if arch not in should_build:
                    should_build.append(arch)

        for msg in sorted(diff_msgs):
            logger.info(" * {}".format(msg))

    if len(should_build):
        logger.info("Arches [{}] should be build".format(
                    ",".join(should_build)))
    else:
        logger.info("No builds are needed.")

    return (should_build, diffs)


def log_serial(manifest_url):
    """
    Read the build-info.txt in and output the serial to the log.
    The location of build-info.txt is calculated relative to the manifest.
    """

    buildinfo_loc = '/'.join([manifest_url.rsplit('/', 1)[0],
                              'unpacked',
                              'build-info.txt'])
    logger.info("Reading build-info.txt %s" % buildinfo_loc)

    if "http" in buildinfo_loc:
        logger.info("     Requesting build-info.txt over http")
        raw_list = bytes.decode(requests.get(buildinfo_loc).content)

    else:
        try:
            with open(buildinfo_loc, 'r') as buildinfo_file:
                raw_list = buildinfo_file.read()
        except IOError:
            raw_list = ''

    for line in raw_list.splitlines():
        if '=' in line:
            key, value = line.split('=')
            if key == 'serial':
                logger.info('Build with serial %s used for manifest' % value)
                return

    logger.info('Serial for build used for manifest was not found')
    return


def get_meta(pattern, suite, pocket, archive, ppas=None, proposed=False):
    """
    Generate locations where the manifests are and fetch them
    """
    pkg_lists = {}
    for arch in iter_archs(suite):
        manifest_url = pattern % {'SUITE': suite,
                                  'ARCH': arch,
                                  'VERSION': get_suite_version(suite)}
        log_serial(manifest_url)

        pkg_lists[arch] = {}
        installed_manifest = get_manifest(manifest_url)
        if installed_manifest is None:
            pkg_lists[arch]['new_build'] = True
        else:
            pkg_lists[arch]['installed'] = installed_manifest
        pkg_lists[arch]['archive'] = get_archive_meta(archive, suite, pocket,
                                                      arch, ppas, proposed)
    return pkg_lists


def write_trigger_file(fns, suite, arch):
    """
    Write a trigger file for Jenkins
    """
    content = []

    if os.path.exists(fns):
        with open(fns, 'r') as f:
            for line in f.readlines():
                content.append(line.strip())

    if 'SUITE=%s' % suite not in content:
        content.append("SUITE=%s" % suite)

    if 'ARCH_%s=build' % arch not in content:
        content.append("ARCH_%s=build" % arch)

    with open(fns, 'w') as f:
        f.write("\n".join(content))
        f.write("\n")


def run(pattern, suite, pocket, host, ppas=None, proposed=None):
    """
    Convenience function for executing the code
    """
    pkg_list = get_meta(pattern, suite, pocket, host, ppas, proposed)
    (should_build, diffs) = compare_meta(pkg_list)
    return (pkg_list, should_build, diffs)



if __name__ == "__main__" and __package__ is None:
    epilog_text = """
Calculate difference between Cloud Image manifest and
what is available in an archive.

Example:
    should_build.py --release --trigger build.trigger --suite trusty

"""

    parser = argparse.ArgumentParser(epilog=epilog_text,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)  # noqa
    parser.add_argument("--suite",
                        action="store",
                        default="trusty",
                        help="Ubuntu release suite")
    parser.add_argument("--arch",
                        action="append",
                        help="Override suite architecture defaults")
    parser.add_argument("--ppa",
                        action="append",
                        help="Look at PPA's too")
    parser.add_argument("--host",
                        action="append",
                        default="archive.ubuntu.com",
                        help="Which host to use for indicies")
    parser.add_argument("--pocket",
                        action="store",
                        default="main",
                        help="Which pocket to explore, i.e. main, universe")
    parser.add_argument("--pattern",
                        action="store",
                        default=DAILY_PATTERN,
                        help="Patterns to get manifests, in Python syntax")
    parser.add_argument("--release",
                        action="store_true",
                        help="Compare against the latest released image")
    parser.add_argument("--daily",
                        action="store_true",
                        help="Use the latest daily")
    parser.add_argument("--trigger",
                        action="store",
                        help="Name of trigger file for Jenkins job to use")
    parser.add_argument("--proposed",
                        action="store_true",
                        required=False,
                        help="Should proposed be considered")
    parser.add_argument("--wait_package",
                        action="append",
                        help="Wait for a certain pacakge(s). Format is <pkg>::<ver>, i.e udev::1.0.1")  # noqa
    parser.add_argument("--wait_time",
                        action="store",
                        type=int,
                        default=180,
                        help="Wait X time for package to become available")

    args = parser.parse_args()

    if args.arch:
        build_arches["override"] = args.arch

    pattern = None
    if args.release:
        pattern = RELEASE_PATTERN
    elif args.daily:
        pattern = DAILY_PATTERN
    else:
        pattern = args.pattern

    ready = False
    wait_pkg_write = False
    while not ready:
        pkg_list, should_build, _ = \
            run(pattern, args.suite, args.pocket, args.host, ppas=args.ppa,
                proposed=args.proposed)

        if args.wait_package:
            ready, not_met = check_pkg_versions(pkg_list, args.wait_package)
            if not ready:
                logger.info("Did not find pkg version waiting on")
                logger.info("Still waiting on: %s" % "\t\n".join(not_met))
                logger.info("Taking a nap for %s" % args.wait_time)
                time.sleep(args.wait_time)
            else:
                wait_pkg_write = True
                logger.info("Found package versions that we're waiting on")
        else:
            ready = True

    if args.trigger:
        for arch in should_build:
            write_trigger_file(args.trigger, args.suite, arch)

        if wait_pkg_write:
            write_trigger_file(args.trigger, args.suite, 'all')