~ubuntu-on-ec2/vmbuilder/automated-ec2-builds-fginther

#!/usr/bin/python3
#
# Extract license information from manifest file
#
# This script is dead simple -- using the pull-lp-source tool, it
# iterates over a manifest and then downloads the source. Then it
# uses the 'license-check' tool to extract the information.
#
#

import __future__
import argparse
import codecs
import csv
import logging as LOG
import json
import os
import os.path
import re
import subprocess
import time
import tempfile

LOG.basicConfig(format='%(levelname)s:%(message)s', level=LOG.DEBUG)

# Populate the system licenses we know about
system_licenses = {}
common_license_d = "/usr/share/common-licenses"
for root, dirs, files in os.walk(common_license_d):
    for f in files:
        LOG.info("Added system license: %s" % f)
        system_licenses[f] = "%s/%s" % (common_license_d, f)

## Shamelessly stolen from cloud-init
def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
         logstring=False):
    if rcs is None:
        rcs = [0]
    try:

        if not logstring:
            LOG.debug(("Running command %s with allowed return codes %s"
                       " (shell=%s, capture=%s)"), args, rcs, shell, capture)
        else:
            LOG.debug(("Running hidden command to protect sensitive "
                       "input/output logstring: %s"), logstring)

        if not capture:
            stdout = None
            stderr = None
        else:
            stdout = subprocess.PIPE
            stderr = subprocess.PIPE
        stdin = subprocess.PIPE
        sp = subprocess.Popen(args, stdout=stdout,
                        stderr=stderr, stdin=stdin,
                        env=env, shell=shell)
        (out, err) = sp.communicate(data)
    except OSError as e:
        LOG.warn("cmd=%s reason=%s" % (args, e))
        return (None, None)

    rc = sp.returncode  # pylint: disable=E1101
    if rc not in rcs:
        LOG.warn("stdout=%s\nstderr=%s\nexit_code=%s\ncmd=%s" % \
                        (out, err, rc, args))
        return (None, None)

    # Just ensure blank instead of none?? (iff capturing)
    if not out and capture:
        out = ''
    if not err and capture:
        err = ''
    return (out, err)

def _normalize_license(lic):
    """
    Take the long name and map to the canonical form, i.e.
        GPL (v1 or later) to GPL
    """
    version = None
    try:
        version = re.search(r".*(\(.*\))", lic).group(1)
        LOG.debug("Reduced license version to: %s" % version)
    except:
        LOG.info("Did not find license version for %s" % lic)

    # Get just the version
    name = lic.split()[0]
    if version:
        version = re.sub(r'(\(|\)|v)', '', version)
        version = version.split()[0]

    # v1 are not shown as "-1"
    if version:
        if version == "1":
            version = None
        elif ".0" in version:
            version = version.split('.0')[0]

    # Handle short names, i.e. GPL
    nam_ver = "%s-%s" % (name, version)
    if not version:
        nam_ver = name

    LOG.debug("Looking for '%s' [%s] in system licenses" % (lic, nam_ver))
    if nam_ver in system_licenses:
        LOG.debug("   found %s as matching license" % nam_ver)
        return (nam_ver, system_licenses[nam_ver])
    else:
        LOG.debug("   did not find %s in system licenses" % nam_ver)

    return (None, None)

def _stripper(string):
    # Match on explicit licences
    if re.search("\/usr\/share\/common-licenses\/([A-z].*)", string):
        string =  re.search("\/usr\/share\/common-licenses\/(.*)", string).group(1)

    # Reg-exes to try and find stuff
    licenses = {
        "(the.BSD.license|BSD.Copyright)": 'BSD',
        "(BSD-1|BSD \(2 clause\))": 'BSD (2 clause)',
        "(BSD-3|BSD \(3 clause\))": 'BSD (3 clause)',
        "(BSD-4|BSD \(4 clause\))": 'BSD (4 clause)',
        "BSD with advertising": 'BSD Advertising',
        "GPL \(unversioned/unknown version\)": 'GPL',
        "GFDL \(unversioned/unknown version\)": 'GFDL',
        "GFDL-NIV \(unversioned/unknown version\)": 'GFDL-NIV',
        "(GFDL-NIV \(v1\)|GFDL-1)": 'GFDL (v1)',
        "(GFDL-NIV \(v2\)|GFDL-2)": 'GFDL (v2)',
        "(GFDL-NIV \(v3\)|GFDL-3)": 'GFDL (v3)',
        "(GFDL-NIV \(v1 or later\)|GNU GFDL, Version 2 or later|GFDL-1\+|GFDL \(1,\))":
            'GFDL-NIV (v1 or later)',
        "(GFDL-NIV \(v2 or later\)|GNU GFDL, Version 2 or later|GFDL-2\+|GFDL \(2,\))":
            'GFDL-NIV (v2 or later)',
        "(GFDL-NIV \(v3 or later\)|GNU GFDL, Version 3 or later|GFDL-3\+|GFDL \(3,\))":
            'GFDL-NIV (v3 or later)',
        "(GFDL \(v1\)|GFDL-1)": 'GFDL (v1)',
        "(GFDL \(v2\)|GFDL-2)": 'GFDL (v2)',
        "(GFDL \(v3\)|GFDL-3)": 'GFDL (v3)',
        "(GFDL \(v1.2 or later\)|GNU GFDL, Version 1.2 or later|GFDL-1.2\+|GFDL \(1.2,\)|GNU Free Documentation License.*1.2.*)":
            'GFDL (v1 or later)',
        "(GFDL \(v1 or later\)|GNU GFDL, Version 1 or later|GFDL-1\+|GFDL \(1,\))":
            'GFDL (v1 or later)',
        "(GFDL \(v2 or later\)|GNU GFDL, Version 2 or later|GFDL-2\+|GFDL \(2,\))":
            'GFDL (v2 or later)',
        "(GFDL \(v3 or later\)|GNU GFDL, Version 3 or later|GFDL-3\+|GFDL \(3,\))":
            'GFDL (v3 or later)',
        "(GPL \(v1\)|GPL-1|GNU General Public License version 1)": 'GPL (v1)',
        "(GPL \(v2\)|GPL-2|GNU General Public License version 2)": 'GPL (v2)',
        "(GPL \(v3\)|GPL-3|GNU General Public License version 3)": 'GPL (v3)',
        "(GPL \(v1 or later\)|GNU GPL, Version 2 or later|GNU General Public License version 1 or later|GNU General Public License version 1, or any later version|GPL-1\+|GPL \(1\,\)|GPL v1 or above|GNU General Public License, either version 1 or.*any newer version|GNU General Public License \(GPL\)\, version 1\, or.*at your option any later version)":
            'GPL (v1 or later)',
        "(GPL \(v2 or later\)|GNU GPL, Version 2 or later|GNU General Public License version 2 or later|GNU General Public License version 2, or any later version|GPL-2\+|GPL \(2\,\)|GPL v2 or above|GNU General Public License, either version 2 or.*any newer version|GNU General Public License \(GPL\)\, version 2\, or.*at your option any later version)":
            'GPL (v2 or later)',
        "(GPL \(v3 or later\)|GNU GPL, Version 3 or later|GNU General Public License version 3 or later|GNU General Public License version 3, or any later version|GPL-3\+|GPL \(3\,\)|GPL v3 or above|GNU General Public License, either version 3 or.*any newer version|GNU General Public License \(GPL\)\, version 3\, or.*at your option any later version)":
            'GPL (v3 or later)',
        "(LGPL \(v2 or later\)|LGP \(2\)|LGPL \(2,\)|GNU LESSER GENERAL PUBLIC LICENSE.*Version 2)":
            'LGPL (v2 or later)',
        "(LGPL \(v2.1 or later\)|LGPL \(v2.1\)|LGPL \(2.1,\)|GNU LESSER GENERAL PUBLIC LICENSE.*Version 2.1)":
            'LGPL (v2.1 or later)',
        "(LGPL \(v3.0 or later\)|LGPL \(v3.0\)|LGPL \(3,\)|GNU LESSER GENERAL PUBLIC LICENSE.*Version 3)":
            'LGPL (v3.0 or later)',
        "(LDP GENERAL PUBLIC LICENSE|LDP General Public License)": "LDPL",
        "(MIT|M.I.T|MIT/X11 \(BSD like\))": 'MIT/X11 (BSD like)',
        "(MIT License \(Expat\)|EXPAT)": 'EXPAT',
        "Apache": 'Apache',
        "CC-BY": 'Creative Commons Attribution',
        "(CC0|Creative Commons Zero)": 'CC0',
        "(CDDL|Common Development and Distribution)": 'CCDL',
        "(CPL|IBM Common Public License)": "CPL",
        "(EFL|Eiffel Forum License)": 'EFL',
        "(Artistic|Artistic-1)": 'Artistic',
        "ISC": 'ISC',
        "WTFPL": "WTFPL",
        "(MPL \(v2.0\)|Mozilla Public License 2.0)": "MPL (v2.0)",
        "(MPL \(v1.0\)|Mozilla Public License 1.0)": "MPL (v1.0)",
        "(MPL \(v1.1\)|Mozilla Public License 1.1)": "MPL (v1.1)",
        "Python License": "Python",
        "QPL": 'QPL',
        "Zope Public License": 'Zope',
    }

    generic_licenses = {
        "GPL": "GPL",
        "BSD": "BSD",
        "Public Domain": "Public Domain",
        "(\*No copyright\* UNKNOWN|\*No copyright\* Public domain)": "Copy-left",
        "Copying and distribution of this file, with or without modification, are permitted in any medium without royalty provided the copyright notice and this notice are preserved.": "Copy-left",
        "You may freely.*binary or source.*": "Copy-left",
        "Public Domain|Copyright\: public-domain": 'Public Domain',
        "Read the file copyright in the main directory of this package for copyright information.": "Parent-license",

    }

    string = string.replace('\n', ' ').strip()
    matched = []
    for k, v in licenses.items():
        if re.search(k, string):
            matched.append(v)

    if len(matched) == 0:
        for k, v in generic_licenses.items():
            if re.search(k, string):
                matched.append(v)

    if len(matched) == 0:
        return ([], True)

    return ([x for x in set(matched)], False)


def parse_manifest(name_f, dump_f, dump_reuse, download=True):
    """
    Iterate over manifest of '<PKG> <VER>' and fetch the files
    via 'pull-lp-source'
    """

    if not os.path.exists(name_f):
        raise Exception("No such file %s" % name_f)

    pkgs = []
    pkgs_names = []
    with codecs.open(name_f, 'r', encoding='utf-8') as f:
        for meta in f.readlines():
            (pkg, version) = meta.split()
            pkgs.append((pkg, version))
            pkgs_names.append(pkg)

    LOG.info("Found %s packages" % len(pkgs))

    # Populate system licenses
    licenses = {'summary': [],
                'system_licenses': system_licenses,
                'pkg_non_standard_licenses': {},
                'pkg_standard_licenses': {},
               }
    for license in system_licenses:
        licenses['pkg_standard_licenses'][license] = []

    # Move the directory
    current_d = os.getcwd()
    existing_paths = []
    my_path = None
    if dump_reuse:
        os.chdir(dump_reuse)
        my_path = dump_reuse
        # Populate the existing
        for root, dirs, _ in os.walk(dump_reuse, topdown=True):
            for dir_n in dirs:
                existing_paths.append(dir_n)
    else:
        tmp_d = tempfile.mkdtemp(prefix="source.", dir=current_d)
        os.chdir(tmp_d)
        my_path = dump_reuse

    # we'll strip this out later, but this makes the licenses easy
    licenses['tmp_path'] = my_path

    def is_existing(pkg):
        """Determine if pkg and version have been downloaded already"""
        for p in existing_paths:
            if pkg in p:
                LOG.info("Already downloaded source for %s" % pkg)
                return True
        return False

    # Download the packages
    lp_cmd = "pull-lp-source"
    for pkg_meta in pkgs:
        (pkg, version) = pkg_meta
        cmd = [lp_cmd, pkg, version]

        if not is_existing(pkg) and download:
            LOG.info("Fetching %s %s" % (pkg, version))
            (out, _err) = subp(cmd, rcs=[0, 1])

    # Now process them
    for root, dirs, _  in os.walk(os.getcwd(), topdown=True):
        for name in dirs:
            name_d = "%s" % os.path.join(root, name)

            if re.search(r'.*(patches|\.pc|packages|tests|debian\-example)',
                         name_d) or \
               re.search(r'.*debian\/(patches|\.pc|packages|tests).*',
                         name_d):
                continue

            control = None
            copyrights = []
            for sr, sd, sf in os.walk(name_d):
                for sfn in sf:
                    candidate_f = os.path.join(sr, sfn)

                    if re.search(r'.*(.git.*|.bzr.*|.py|.sh|.c|.h|.patch|CVS)$',
                                 candidate_f) or \
                       re.search(r'.*\/licenses\/.*', candidate_f):
                       continue
                    elif re.match('.*\/control$', candidate_f):
                        control = candidate_f
                    elif re.match('.*(copyright.*|LICENSE.*|WHENCE|COPYING.*|copying.*|license.*)',
                                  candidate_f):
                        copyrights.append(candidate_f)

            sub_p = []
            sub_b = []
            sub_a = []
            src_p = None

            if not control:
                continue

            with codecs.open(control, 'r', encoding='utf-8') as f:
                for line in f.readlines():
                    if re.match('^Source:.*', line):
                        src_p = line.split(':')[-1].strip()
                    if re.match('^Package:.*', line):
                        _pkg = line.split(':')[-1].strip()
                        if _pkg in pkgs_names:
                            sub_p.append(_pkg)
                        else:
                            sub_b.append(_pkg)
                        sub_a.append(_pkg)

            if src_p not in licenses:
                licenses[src_p] = {'licenses': {}, 'standard_licenses': {}}

            for copyright in copyrights:
                cmd = ["licensecheck", "--lines=500", copyright]
                (out, _err) =  subp(cmd)

                out = str(out) #, encoding='utf-8')
                cc = (out.strip()).split(': ')[-1]
                ccn = copyright
                _found, _unknown = _stripper(cc)

                if _unknown:
                    with codecs.open(copyright, 'r', encoding='utf-8') as f:
                        _found, _unknown = _stripper(f.read())

                if _unknown:
                    _found = ["UNKNOWN"]

                for lic in _found:
                    sys_license, sys_path = _normalize_license(lic)
                    if my_path is not None:
                    	short_name = lic.replace(my_path, '')
                    else:
                        short_name = lic

                    if not sys_license and not sys_path:
                        if ccn not in licenses[src_p]['licenses']:
                            licenses[src_p]['licenses'][ccn] = []
                        licenses[src_p]['licenses'][ccn].append(short_name)
                        licenses[src_p]['licenses'][ccn] = [
                            x for x in set(licenses[src_p]['licenses'][ccn])]

                        if src_p not in licenses['pkg_non_standard_licenses']:
                            licenses['pkg_non_standard_licenses'][src_p] = {}

                        for k, v in licenses[src_p]['licenses'].items():
                            licenses['pkg_non_standard_licenses'][src_p][k] = v

                    else:
                        if ccn not in licenses[src_p]['standard_licenses']:
                            licenses[src_p]['standard_licenses'][ccn] = []
                        licenses[src_p]['standard_licenses'][ccn].append(
                             sys_license)
                        licenses[src_p]['standard_licenses'][ccn] = [
                            x for x in set(licenses[src_p]['standard_licenses'][ccn])]

                        licenses['pkg_standard_licenses'][sys_license].extend(sub_p)
                        licenses['pkg_standard_licenses'][sys_license] = \
                            sorted([x for x in set(\
                                    licenses['pkg_standard_licenses'][sys_license]
                                  )])

            licenses[src_p]['pkgs_installed'] = sub_p
            licenses[src_p]['pkgs_built_from_source'] = sub_a

            if len(sub_b) > 0:
                licenses[src_p]['pkgs_not_installed'] = sub_b

    os.chdir(current_d)
    if '.json' not in dump_f:
        dump_f += ".json"

    with codecs.open(dump_f, 'w', encoding='utf-8') as f:
        f.write(json.dumps(licenses, indent=4))

    return licenses


def license_html(licenses, manifest, dump_f, version, subversion):
    """
    This generates a standard HTML file with the list of licenseses,
    the text and the package name
    """
    bad_path = licenses['tmp_path']

    text = """
<!DOCTYPE html>
<html>
<head>
<title>License information for Ubuntu Server %s</title>
</head>

<body>
<h1>License information for Ubuntu Server %s for %s</h1>
<h3>Report generated on %s</h3>

<p>Ubuntu is covered by the following polices</p>
<ul>
    <li><p><a href="http://www.canonical.com/intellectual-property-rights-policy">http://www.canonical.com/intellectual-property-rights-policy<a/></li>
    <li><a href="http://www.ubuntu.com/legal">http://www.ubuntu.com/legal</a></li>
    <li>Open Source licenses as disclosed below</li>
</ul>

<p><b>Disclaimer:</b> The author(s) of this tool make no representation or warranty as to the accuracy or completeness of this list or the information it represents, and we expressly disclaim any implied warranties to the fullest extent permissible at law. We accept no liability for any damage or loss arising form your use or reliance on the list. Any and all uses of this list is subject to these terms.</p>

<p>This report was generated using tool which relies on the meta-data for each file. The user of this report is responsible for verifying the accuracy of this list.</p>

<p>This report is only valid for the packages and the versions listed. Since this report is generated from source, it may contain licenses not found in the final packaged version. Again, it is up to the users to verify the veracity of the information.</p>

<h1>Package list</h1>
<p>This document discloses the license information for the following packages and version</p>

<ul>
""" % (version, version, subversion, time.strftime("%d %b %Y"))

    # Copy in the manifest
    LOG.info("Adding manifest to HTML")
    with codecs.open(manifest, 'r', encoding='utf-8') as f:
        for line in f.readlines():
            text += "   <li>%s</li>" % line.strip()
    text += "\n</ul>\n"
    text += """
<hr>
<h1>Standard System Licenses</h1>
<p>These are the standard licenses, which use Debian standard system licenses.</p>

"""
    LOG.info("Processing standard licenses")
    for std_license in licenses['pkg_standard_licenses']:
        pkgs = licenses['pkg_standard_licenses'][std_license]
        npkgs = len(pkgs)
        if npkgs == 0:
            LOG.info("   Skipping %s (no pkgs)" % std_license)
            continue

        LOG.info("   Adding %s" % std_license)
        LOG.info("       Packages licensed %s" % npkgs)
        license_f = "%s/%s" % (common_license_d, std_license)
        license_txt = None
        with codecs.open(license_f, 'r', encoding='utf-8') as f:
            license_txt = f.read()
        LOG.info("       Read %s" % license_f)

        text += """
    <h2>License: %s</h2>
    <h3>The following packages are covered under this license</h3>
    <ul>
""" % std_license

        for pkg in pkgs:
            text += "        <li>%s</li>\n" % pkg.strip()

        text += """
    </ul>
    <h3>License Text:</h3>
    <pre>
%s
    </pre>
""" % license_txt

    text += """
<hr>
<h1>Non-standard Licenses</h1>
<p>These license are free and open licenses, but are either modified,
or are licenses that allow modification to their terms.</p>
"""

    for pkg in licenses['pkg_non_standard_licenses']:
        text += "      <h2>Package: %s</h2>" % pkg
        LOG.info("Adding licenses for %s" % pkg)
        pkg_license = licenses['pkg_non_standard_licenses'][pkg]
        for nstd_l in pkg_license:
            license_txt = None
            LOG.info("  Adding %s as text" % nstd_l)

            try:
                with codecs.open(nstd_l, 'r', encoding='utf-8') as f:
                    license_txt = f.read()
            except:
                continue

            text += """
<h3>License(s): %s"</h3>
<pre>
%s
</pre>""" % (nstd_l.replace("%s/" % bad_path, ''), license_txt)

    text += """
</body>
</html>
"""

    # Write the output html
    if ".html" not in dump_f:
        dump_f += ".html"
    with codecs.open(dump_f, 'w', encoding='utf-8') as f:
        f.write(text)


if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('--file',
            required=True,
            default=False,
            help="File to read manifest from")
    parser.add_argument('--out',
            required=True,
            default=False,
            help="Place to dump the output")
    parser.add_argument('--reuse',
            required=False,
            default=None,
            help="Reuse existing downloads")
    parser.add_argument('--no_download',
            action="store_false",
            required=False,
            default=True,
            help="Don't download, use with --reuse")
    parser.add_argument('--version',
            action="store",
            required=False,
            default="12.04",
            help="Version of Ubuntu")
    parser.add_argument("--subversion",
            action="store",
            required=True,
            help="Derivative information")


    opts = parser.parse_args()
    licenses = parse_manifest(opts.file, opts.out, opts.reuse,
                              opts.no_download)
    license_html(licenses, opts.file, opts.out, opts.version,
                 opts.subversion)