~nskaggs/juju-release-tools/generate-release-notes

#!/usr/bin/python
"""Script for building source and binary debian packages."""

from __future__ import print_function

from argparse import ArgumentParser
from collections import namedtuple
import os
import re
import shutil
import subprocess
import sys

__metaclass__ = type


DEBS_NOT_FOUND = 3

# This constant defines the location of the base source package branch.
DEFAULT_SPB = 'lp:~juju-qa/juju-release-tools/packaging-juju-core-default'
DEFAULT_SPB2 = 'lp:~juju-qa/juju-release-tools/packaging-juju-core2-default'


# This constant defines the status of the series supported by CI and Releases.
SUPPORTED_RELEASES = """\
12.04 precise LTS
12.10 quantal HISTORIC
13.10 saucy HISTORIC
14.04 trusty LTS
14.10 utopic HISTORIC
15.04 vivid HISTORIC
15.10 wily HISTORIC
16.04 xenial LTS
16.10 yakkety SUPPORTED
17.04 zesty DEVEL
"""


SourceFile = namedtuple('SourceFile', ['sha256', 'size', 'name', 'path'])


CREATE_LXC_TEMPLATE = """\
set -eu
sudo lxc-create -t ubuntu-cloud -n {container} -- -r {series} -a {arch}
sudo mkdir /var/lib/lxc/{container}/rootfs/workspace
echo "lxc.mount.entry = {build_dir} workspace none bind 0 0" |
    sudo tee -a /var/lib/lxc/{container}/config
"""


BUILD_DEB_TEMPLATE = """\
sudo lxc-attach -n {container} -- bash <<"EOT"
    set -eu
    echo "\nInstalling common build deps.\n"
    cd workspace
    # Wait for Cloud-init to complete to indicate the machine is in a ready
    # state with network to do work,
    while ! tail -1 /var/log/cloud-init-output.log | \
            egrep -q -i 'Cloud-init .* finished'; do
        echo "Waiting for Cloud-init to finish."
        sleep 5
    done
    set +e
    # The cloud-init breaks arm64, ppc64el. s390x /etc/apt/sources.list.
    if [[ $(dpkg --print-architecture) =~ ^(arm64|ppc64el|s390x)$ ]]; then
        sed -i \
            -e 's,archive.ubuntu.com/ubuntu,ports.ubuntu.com/ubuntu-ports,' \
            /etc/apt/sources.list
    fi
    # Adding the ppa directly to sources.list without the archive key
    # requires apt to be run with --force-yes
    echo "{ppa}" >> /etc/apt/sources.list
    export DEBIAN_FRONTEND=noninteractive
    apt-get update
    apt-get install -y --force-yes build-essential devscripts equivs
EOT
sudo lxc-attach -n {container} -- bash <<"EOT"
    set -eux
    echo "\nInstalling build deps from dsc.\n"
    cd workspace
    export DEBIAN_FRONTEND=noninteractive
    mk-build-deps -i --tool 'apt-get --yes --force-yes' *.dsc
EOT
sudo lxc-attach -n {container} -- bash <<"EOT"
    set -eux
    echo "\nBuilding the packages.\n"
    cd workspace
    rm *build-deps*.deb || true
    dpkg-source -x *.dsc
    cd $(basename *.orig.tar.gz .orig.tar.gz | tr _ -)
    dpkg-buildpackage -us -uc
EOT
"""


CREATE_SPB_TEMPLATE = """\
set -eux
bzr branch {branch} {spb}
cd {spb}
bzr import-upstream {version} {tarfile_path}
bzr merge . -r upstream-{version}
bzr commit -m "Merged upstream-{version}."
"""


BUILD_SOURCE_TEMPLATE = """\
set -eux
bzr branch {spb} {source}
cd {source}
dch --newversion {ubuntu_version} -D {series} --force-distribution "{message}"
debcommit
bzr bd -S -- -us -uc
"""

DEBSIGN_TEMPLATE = 'debsign -p {gpgcmd} *.changes'


UBUNTU_VERSION_TEMPLATE_EPOCH = \
    '{epoch}:{version}-0ubuntu1~{release}.{upatch}~juju1'
DAILY_VERSION_TEMPLATE_EPOCH = \
    '{epoch}:{version}-{date}+{build}+{revid}~{release}'
UBUNTU_VERSION_TEMPLATE = '{version}-0ubuntu1~{release}.{upatch}~juju1'
DAILY_VERSION_TEMPLATE = '{version}-{date}+{build}+{revid}~{release}'


VERSION_PATTERN = re.compile('(\d+)\.(\d+)\.(\d+)')


Series = namedtuple('Series', ['version', 'name', 'status'])


class _JujuSeries:

    LIVING_STATUSES = ('DEVEL', 'SUPPORTED', 'LTS')

    def __init__(self):
        self.all = {}
        for line in SUPPORTED_RELEASES.splitlines():
            series = Series(*line.split())
            self.all[series.name] = series

    def get_devel_version(self):
        for series in self.all.values():
            if series.status == 'DEVEL':
                return series.version
        else:
            raise AssertionError(
                "SUPPORTED_RELEASES is missing the DEVEL series")

    def get_living_names(self):
        return sorted(s.name for s in self.all.values()
                      if s.status in self.LIVING_STATUSES)

    def get_name(self, version):
        for series in self.all.values():
            if series.version == version:
                return series.name
        else:
            raise KeyError("'%s' is not a known series" % version)

    def get_name_from_package_version(self, package_version):
        """Return the series name associated with the package version.

        The series is matched to the series version commonly embedded in
        backported package versions. Official juju package versions always
        contain the series version to indicate the tool-chain used to build.

        Ubuntu devel packages do not have series versions, they cannot be
        matched to a series. As Ubuntu packages are not built with ideal rules,
        they are not suitable for building agents.
        """
        for series in self.all.values():
            if series.version in package_version:
                return series.name
        return None

    def get_version(self, name):
        return self.all[name].version


juju_series = _JujuSeries()


def parse_dsc(dsc_path, verbose=False):
    """Return the source files need to build a binary package."""
    there = os.path.dirname(dsc_path)
    dsc_name = os.path.basename(dsc_path)
    dcs_source_file = SourceFile(None, None, dsc_name, dsc_path)
    files = [dcs_source_file]
    with open(dsc_path) as f:
        content = f.read()
    found = False
    for line in content.splitlines():
        if found and line.startswith(' '):
            data = line.split()
            data.append(os.path.join(there, data[2]))
            files.append(SourceFile(*data))
            if verbose:
                print("Found %s" % files[-1].name)
        elif found:
            # All files were found.
            break
        if not found and line.startswith('Checksums-Sha256:'):
            found = True
    return files


def setup_local(location, series, arch, source_files, verbose=False):
    """Create a directory to build binaries in.

    The directoy has the source files required to build binaries.
    """
    build_dir = os.path.abspath(
        os.path.join(location, 'juju-build-{}-{}'.format(series, arch)))
    if verbose:
        print('Creating %s' % build_dir)
    os.makedirs(build_dir)
    for sf in source_files:
        dest_path = os.path.join(build_dir, sf.name)
        if verbose:
            print('Copying %s to %s' % (sf.name, build_dir))
        shutil.copyfile(sf.path, dest_path)
    return build_dir


def setup_lxc(series, arch, build_dir, verbose=False):
    """Create an LXC container to build binaries.

    The local build_dir with the source files is bound to the container.
    """
    container = '{}-{}'.format(series, arch)
    lxc_script = CREATE_LXC_TEMPLATE.format(
        container=container, series=series, arch=arch, build_dir=build_dir)
    if verbose:
        print('Creating %s container' % container)
    output = subprocess.check_output([lxc_script], shell=True)
    if verbose:
        print(output)
    return container


def build_in_lxc(container, build_dir, ppa=None, verbose=False):
    """Build the binaries from the source files in the container."""
    returncode = 1
    if ppa:
        path = ppa.split(':')[1]
        series = container.split('-')[0]
        ppa = 'deb http://ppa.launchpad.net/{}/ubuntu {} main'.format(
            path, series)
    else:
        ppa = '# No PPA added.'
    # The work in the container runs as a different user. Care is needed to
    # ensure permissions and ownership are correct before and after the build.
    os.chmod(build_dir, 0o777)
    subprocess.check_call(['sudo', 'lxc-start', '-d', '-n', container])
    try:
        build_script = BUILD_DEB_TEMPLATE.format(container=container, ppa=ppa)
        returncode = subprocess.call([build_script], shell=True)
    finally:
        subprocess.check_call(['sudo', 'lxc-stop', '-n', container])
        user = os.environ.get('USER', 'jenkins')
        subprocess.check_call(['sudo', 'chown', '-R', user, build_dir])
        os.chmod(build_dir, 0o775)
    return returncode


def teardown_lxc(container, verbose=False):
    """Destroy the lxc container."""
    if verbose:
        print('Deleting the lxc container %s' % container)
    subprocess.check_call(['sudo', 'lxc-destroy', '-n', container])


def move_debs(build_dir, location, verbose=False):
    """Move the debs from the build_dir to the location dir."""
    found = False
    files = [f for f in os.listdir(build_dir) if f.endswith('.deb')]
    for file_name in files:
        file_path = os.path.join(build_dir, file_name)
        dest_path = os.path.join(location, file_name)
        if verbose:
            print("Found %s" % file_name)
        shutil.move(file_path, dest_path)
        found = True
    return found


def build_binary(dsc_path, location, series, arch, ppa=None, verbose=False):
    """Build binary debs from a dsc file."""
    # If location is remote, setup remote location and run.
    source_files = parse_dsc(dsc_path, verbose=verbose)
    build_dir = setup_local(
        location, series, arch, source_files, verbose=verbose)
    container = setup_lxc(series, arch, build_dir, verbose=verbose)
    try:
        build_in_lxc(container, build_dir, ppa=ppa, verbose=verbose)
    finally:
        teardown_lxc(container, verbose=False)
    found = move_debs(build_dir, location, verbose=verbose)
    if not found:
        return DEBS_NOT_FOUND
    return 0


def create_source_package_branch(build_dir, version, tarfile, branch):
    """Create a new source package branch with the imported release tarfile.

    The new source package can be pushed to a permanent location if it will
    be used as the base for future packages.

    :param build_dir: The root directory to create the new branch in.
    :param version: The upstream version, which is not always the version
        in the tarfile name.
    :param tarfile: The path to the tarfile to import.
    :param branch: The base source package branch to fork and import into.
    :return: The path to the source package branch.
    """
    spb = os.path.join(build_dir, 'spb')
    tarfile_path = os.path.join(build_dir, tarfile)
    script = CREATE_SPB_TEMPLATE.format(
        branch=branch, spb=spb, tarfile_path=tarfile_path, version=version)
    subprocess.check_call([script], shell=True, cwd=build_dir)
    return spb


def make_ubuntu_version(series, version, upatch=1,
                        date=None, build=None, revid=None, epoch=None):
    """Return an Ubuntu package version.

    :param series: The series codename.
    :param version: The upstream version.
    :param upatch: The package patch number for cases where a packaging rules
        are updated and the package rebuilt.
    :param date: The date of the build.
    :param build: The build number in CI.
    :param revid: The revid hash of the source.
    :param epoch: The epoch to pass in version name
    :return: An Ubuntu version string.
    """
    release = juju_series.get_version(series)
    # if daily params are set, we make daily build
    if all([date, build, revid]):
        if epoch:
            return DAILY_VERSION_TEMPLATE_EPOCH.format(
                epoch=epoch, version=version, release=release, upatch=upatch,
                date=date, build=build, revid=revid)
        return DAILY_VERSION_TEMPLATE.format(
            version=version, release=release, upatch=upatch,
            date=date, build=build, revid=revid)

    else:
        if epoch:
            return UBUNTU_VERSION_TEMPLATE_EPOCH.format(
                epoch=epoch, version=version, release=release, upatch=upatch)
        return UBUNTU_VERSION_TEMPLATE.format(
            version=version, release=release, upatch=upatch)


def make_changelog_message(version, bugs=None):
    """Return a changelog message for the version.

    :param version: The upstream version.
    :param bugs: A list of Lp bug numbers or None. They will be formatted
        for the changelog.
    :return: a changelog message.
    """
    match = VERSION_PATTERN.match(version)
    if match is None:
        message = 'New upstream devel release.'
    elif match.group(3) == '0':
        message = 'New upstream stable release.'
    else:
        message = 'New upstream stable point release.'
    if bugs:
        fixes = ', '.join(['LP #%s' % b for b in bugs])
        message = '%s (%s)' % (message, fixes)
    return message


def make_deb_shell_env(debemail, debfullname):
    """Return a replacement environ suitable for DEB building.

    :param debemail: The email address to attribute the changelog entry to.
    :param debfullname: The name to attribute the changelog entry to.
    :return: A modified copy of os.environ that can be passed to subprocesses.
    """
    env = dict(os.environ)
    env['DEBEMAIL'] = debemail
    env['DEBFULLNAME'] = debfullname
    return env


def sign_source_package(source_dir, gpgcmd, debemail, debfullname):
    """Sign a source package.

    The debemail and debfullname must match the identity used in the changelog
    and the changes file.

    :param source_dir: The source package directory.
    :param gpgcmd: The path to a gpg signing command to sign with.
    :param debemail: The email address to attribute the changelog entry to.
    :param debfullname: The name to attribute the changelog entry to.
    """
    env = make_deb_shell_env(debemail, debfullname)
    script = DEBSIGN_TEMPLATE.format(gpgcmd=gpgcmd)
    subprocess.check_call([script], shell=True, cwd=source_dir, env=env)


def create_source_package(source_dir, spb, series, version,
                          upatch='1', bugs=None, gpgcmd=None, debemail=None,
                          debfullname=None, verbose=False,
                          date=None, build=None, revid=None, epoch=None):
    """Create a series source package from a source package branch.

    The new source package can be used to create series source packages.

    :param source_dir: The source package directory.
    :param spb: The path (or location) of the source package branch to create
        the new source package with.
    :param series: The series codename.
    :param version: The upstream version.
    :param upatch: The package patch number for cases where a packaging rules
        are updated and the package rebuilt.
    :param bugs: A list of Lp bug numbers for the changelog or None.
    :param gpgcmd: The path to a gpg signing command to sign with.
        Source packages will be signed when gpgcmd is not None.
    :param debemail: The email address to attribute the changelog entry to.
    :param debfullname: The name to attribute the changelog entry to.
    :param verbose: Increase the information about the work performed.
    :param date: The date of the build.
    :param build: The build number in CI.
    :param revid: The revid hash of the source.
    :param epoch: The epoch to pass in version name
    """

    ubuntu_version = make_ubuntu_version(series, version, upatch,
                                         date, build, revid, epoch)
    message = make_changelog_message(version, bugs=bugs)
    source = os.path.join(source_dir, 'source')
    env = make_deb_shell_env(debemail, debfullname)
    script = BUILD_SOURCE_TEMPLATE.format(
        spb=spb, source=source, series=series, ubuntu_version=ubuntu_version,
        message=message)
    subprocess.check_call([script], shell=True, cwd=source_dir, env=env)
    if gpgcmd:
        sign_source_package(source_dir, gpgcmd, debemail, debfullname)


def build_source(tarfile_path, location, series, bugs,
                 debemail=None, debfullname=None, gpgcmd=None,
                 branch=None, upatch=1, verbose=False,
                 date=None, build=None, revid=None, epoch=None):
    """Build one or more series source packages from a new release tarfile.

    The packages are unsigned by default, but providing the path to a gpgcmd,
    the dsc file will be signed.

    :param tarfile_path: The path to the upstream tarfile. to import.
    :param location: The path to the directory to build packages in.
    :param series: The series codename or list of series codenames.
    :param bugs: A list of Lp bug numbers the release fixes.
    :param gpgcmd: The path to a gpg signing command to sign with.
        Source packages will be signed when gpgcmd is not None.
    :param debemail: The email address to attribute the changelog entry to.
    :param debfullname: The name to attribute the changelog entry to.
    :param branch: The path (or location) of the source package branch to
        create the new source package with.
    :param upatch: The package patch number for cases where a packaging rules
        are updated and the package rebuilt.
    :param verbose: Increase the verbostiy of output.
    :param date: The date of the build.
    :param build: The build number in CI.
    :param revid: The revid hash of the source.
    :param epoch: The epoch to pass in version name
    :return: the exit code (which is 0 or else an exception was raised).
    """
    if not isinstance(series, list):
        series = [series]
    tarfile_name = os.path.basename(tarfile_path)
    version = tarfile_name.split('_')[-1].replace('.tar.gz', '')
    if all([date, build, revid]):
        daily_version = '{}~{}~{}~{}'.format(version, date, build, revid)
        daily_tarfile_name = tarfile_name.replace(version, daily_version)
        tarfile_dir = os.path.dirname(tarfile_path)
        daily_tarfile_path = os.path.join(tarfile_dir, daily_tarfile_name)
        os.rename(tarfile_name, daily_tarfile_name)
        tarfile_path = daily_tarfile_path
        tarfile_name = daily_tarfile_name
        version = daily_version

    files = [SourceFile(None, None, tarfile_name, tarfile_path)]
    spb_dir = setup_local(
        location, 'any', 'all', files, verbose=verbose)
    spb = create_source_package_branch(spb_dir, version, tarfile_name, branch)
    for a_series in series:
        build_dir = setup_local(location, a_series, 'all', [], verbose=verbose)
        create_source_package(
            build_dir, spb, a_series, version,
            upatch=upatch, bugs=bugs, gpgcmd=gpgcmd,
            debemail=debemail, debfullname=debfullname, verbose=verbose,
            date=date, build=build, revid=revid, epoch=epoch)
    return 0


def print_series_info(package_version=None):
    exitcode = 1
    if package_version:
        version = juju_series.get_name_from_package_version(package_version)
        if version:
            print(version)
            return 0
    return exitcode


def main(argv):
    """Execute the commands from the command line."""
    exitcode = 0
    args = get_args(argv)
    if args.command == 'source':
        exitcode = build_source(
            args.tar_file, args.location, args.series, args.bugs,
            debemail=args.debemail, debfullname=args.debfullname,
            gpgcmd=args.gpgcmd, branch=args.branch, upatch=args.upatch,
            verbose=args.verbose, date=args.date, build=args.build,
            revid=args.revid, epoch=args.epoch)
    elif args.command == 'binary':
        exitcode = build_binary(
            args.dsc, args.location, args.series, args.arch,
            ppa=args.ppa, verbose=args.verbose)
    elif args.command == 'print':
        exitcode = print_series_info(
            package_version=args.series_name_from_package_version)
    return exitcode


def get_args(argv=None):
    """Return the arguments for this program."""
    parser = ArgumentParser("Build debian packages.")
    parser.add_argument(
        "-v", "--verbose", action="store_true", default=False,
        help="Increase the verbosity of the output")
    subparsers = parser.add_subparsers(help='sub-command help', dest="command")
    src_parser = subparsers.add_parser('source', help='Build source packages')
    src_parser.add_argument(
        '--debemail', default=os.environ.get("DEBEMAIL"),
        help="Your email address; Environment: DEBEMAIL.")
    src_parser.add_argument(
        '--debfullname', default=os.environ.get("DEBFULLNAME"),
        help="Your full name; Environment: DEBFULLNAME.")
    src_parser.add_argument(
        '--epoch', default=None, help="The epoch for package version")
    src_parser.add_argument(
        '--gpgcmd', default=None,
        help="Path to a gpg signing command to make signed packages.")
    src_parser.add_argument(
        '--branch', help="The base/previous source package branch.")
    src_parser.add_argument(
        '--upatch', default='1', help="The Ubuntu patch number.")
    src_parser.add_argument('tar_file', help="The release tar file.")
    src_parser.add_argument("location", help="The location to build in.")
    src_parser.add_argument(
        'series', help="The destination Ubuntu release or LIVING for all.")
    src_parser.add_argument(
        '--date', default=None, help="A datestamp to apply to the build")
    src_parser.add_argument(
        '--build', default=None, help="The build number from CI")
    src_parser.add_argument(
        '--revid', default=None, help="The short hash for revid")
    src_parser.add_argument(
        'bugs', nargs='*', help="Bugs this version will fix in the release.")
    bin_parser = subparsers.add_parser('binary', help='Build a binary package')
    bin_parser.add_argument(
        '--ppa', default=None, help="The PPA that provides package deps.")
    bin_parser.add_argument("dsc", help="The dsc file to build")
    bin_parser.add_argument("location", help="The location to build in.")
    bin_parser.add_argument("series", help="The series to build in.")
    bin_parser.add_argument("arch", help="The dpkg architecture to build in.")
    print_parser = subparsers.add_parser('print', help='Print series info')
    print_parser.add_argument(
        '--series-name-from-package-version',
        help="Print the series name associated with the package version.")
    args = parser.parse_args(argv[1:])
    if getattr(args, 'series', None) and args.series == 'LIVING':
        args.series = juju_series.get_living_names()
    if args.command == 'source' and args.branch is None:
        tarfile_name = os.path.basename(args.tar_file)
        version = tarfile_name.split('_')[-1].replace('.tar.gz', '')
        if version.startswith('2.'):
            args.branch = DEFAULT_SPB2
        else:
            args.branch = DEFAULT_SPB
    return args


if __name__ == '__main__':
    sys.exit(main(sys.argv))