~nskaggs/ubuntu-filemanager-app/fix-ap-item-renames

#!/usr/bin/python3

# Copyright (C) 2015 Stefano Verzegnassi <verzegnassi.stefano@gmail.com>
# Copyright (C) 2015 Didier Roche <didrocks@ubuntu.com> 
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License version 3, as published
# by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranties of
# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
# PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.

#
# Source code at:
# https://code.launchpad.net/~verzegnassi-stefano/+junk/get-click-deps
#
# A script to automate the fetching all the external dependencies of a
# Click packaged application from internet, and copying them in a given
# folder, so that the click packaging tool can easily build them in the
# package.
#
# usage: get-click-deps [-h] [-f] [-d] [-e] [-c SCRIPT_PATH]
#                       manifest_path {amd64,i386,armhf} target_path
#
# A tool for adding external libraries to a Ubuntu SDK application or scope.
#
# positional arguments:
#   manifest_path         path of json file containing the list of packages to
#                         be downloaded.
#   {amd64,i386,armhf}    CPU architecture ("amd64", "i386" or "armhf")
#   target_path           path to the target (a click package or a folder)
#                         where this tool will include the downloaded binaries.
#                         If the folder does not exist, it will be created.
#
# optional arguments:
#   -h, --help            show this help message and exit
#   -f, --force-download  force a new download of the packages
#   -d, --delete-temp     delete temp files at the end of the process
#   -e, --extract-only    only create temp directory and extract the content of
#                         downloaded packages
#   -c SCRIPT_PATH, --custom-script SCRIPT_PATH
#                         run a custom script after the extraction of Debian
#                         packages and before copying their content to the
#                         target destination. The tool will execute the script
#                         with the path to the packages dump as argument. If
#                         the '-e' flag has been specified, the script will be
#                         anyway executed. This option is useful when you need
#                         to automatically modify the content of temp folder
#                         (e.g. when you need to fix some path before including
#                         the files in a click package).
#
#
# USAGE EXAMPLE:
# get-click-deps packages.json armhf <path/to/package.click>
#
# package.json is the package manifest, and contains all the references to the
# .deb packages to be included into the Ubuntu SDK project.
# If you're familiar to the Debian/Ubuntu world, you'll see that it's pretty
# similar to therepository management of these distros:
# see https://wiki.debian.org/SourcesList for further informations.
#
# An example of packages.json file is:
# {
#     "armhf": [
#         {
#             "url": "http://ports.ubuntu.com/ubuntu-ports/",
#             "dist": "vivid",
#             "component": "main",
#             "packages": [
#                 "libgl1-mesa-glx",
#                 "libxslt1.1",
#                 "libxcb-glx0",
#                 "libxcb-dri3-0",
#                 "libxcb-present0",
#                 "libxshmfence1",
#                 "libxxf86vm1"
#             ]
#         },
#
#         {
#             "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu",
#             "dist": "vivid",
#             "component": "main",
#             "packages": [
#                 "libreoffice-vanilla"
#             ]
#         }
#     ],
#
#     "amd64": [
#             "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu",
#             "dist": "vivid",
#             "component": "main",
#             "packages": [
#                 "libreoffice-vanilla"
#             ]
#         }
#     ]
# }
#
# Instead of a click package, you can specify the build folder used by Ubuntu
# SDK for compiling the sources of your project. This way the binaries
# downloaded by this tool will be automatically included in the .click package
# the next time you ask Ubuntu SDK to create a new package.
#

# TODO: Complete error handling
# TODO: Make target_path optional if the '-e' flag has been specified.

import sys
import os
import stat
import time
import argparse
import json
import gzip
import subprocess
import urllib.request, urllib.error, urllib.parse

def get_timestamp():
    return time.time()


def check_internet_connection():
    try:
        urllib.request.urlopen('http://www.launchpad.net', timeout=20)
        return True
    except urllib2.error.URLError as err:
        pass
    return False


def get_arch_triplet(arch):
    return subprocess.check_output([
        'dpkg-architecture',
        '-A',
        arch,
        '-qDEB_TARGET_MULTIARCH'])


def download_file(url, dest, verbose=True):
    if verbose:
        print ("\nDownloading:\n{}".format(url))
    # TODO: Switch to subprocess
    os.system('cd %s && { curl -# -O %s ; cd - ; }' % (dest, url))


def download_file_and_rename(url, dest, new_filename, verbose=True):
    if verbose:
        print ("\nDownloading:\n{}".format(url))
    new_path = os.path.join(dest, new_filename)
    subprocess.call(['curl', url, '-#', '-o', new_path])

    return new_path


def get_package_download_url(package_name, packages_list, base_url):
    pkgs_list = packages_list.decode('utf-8').split('\n')
    index = pkgs_list.index('Package: %s' % package_name)

    for i in range(index, len(pkgs_list)):
        if pkgs_list[i].find('Filename:') > -1:
            return "%s/%s" % (base_url, pkgs_list[i].replace('Filename: ', ''))


def get_URLs_for_arch(manifest_path, arch, destpath):
    urls = []

    f = open(manifest_path)
    content = json.load(f)
    f.close()

    # Download repository index for each repository in the JSON package
    # manifest.
    try:
        repo_index = 0
        for repo in content[arch]:
            repo_index_url = '%s/dists/%s/%s/binary-%s/Packages.gz' % (
                repo['url'],
                repo['dist'],
                repo['component'],
                arch)

            print ("\nDownloading repository index at:\n".format(repo_index_url))

            repo_index_zip = download_file_and_rename(
                repo_index_url,
                destpath,
                'repo-index-%s-%s.gz' % (arch, repo_index),
                False)

            with gzip.open(repo_index_zip, 'r') as f:
                repo_index_content = f.read()
                f.close()

            # Get the download URL of each package of the repository.
            packages = repo['packages']
            for package in packages:
                urls.append(get_package_download_url(
                    package,
                    repo_index_content,
                    repo['url']))

            repo_index += 1
    except KeyError:
        # Arch not found in the manifest. Exit with no error, since there's no
        # need to run the script for this arch.
        print ("\n\nRequested arch has been not specified in the manifest. \
            Exiting...")
        sys.exit(0)
    print ("\nObtained packages informations")
    return urls


def check_if_temp_folder_already_exists(path):
    return os.path.isdir(path)


def copy_directory_content(sourcepath, destpath):
    subprocess.call(['cp', '-r', '%s/.' % sourcepath, destpath])


def delete_folder(path, recursive=False):
    if recursive:
        flag = '-rf'
    else:
        flag = '-f'
    subprocess.call(['rm', flag, path])


def extract_deb_package(deb_path, destpath):
    subprocess.call(['dpkg-deb', '-x', deb_path, destpath])


def extract_click_package(click_path, destpath):
    extract_deb_package(click_path, destpath)

    manifest = subprocess.check_output(['click', 'info', click_path])

    # The manifest we get has an 'installed-size' key with the value of the
    # previous package. Anyway this value will be replaced when we'll run
    # 'click build <pkg>', so there's no reason for removing it here.
    f = open(os.path.join(destpath, 'manifest.json'), 'w')
    f.write(manifest)
    f.close()

    return destpath


def build_click_package(source_dirpath):
    output = subprocess.check_output(['click', 'build', source_dirpath])

    for line in output.split(os.linesep):
        if line.find('Successfully built package in ') > -1:
            # FIXME: Very ugly.
            path = line.replace('Successfully built package in \'', '')
            path = path.replace('\'.', '')
            return path


def copy_file(sourcepath, destpath, overwrite=False):
    flag = ''

    if not overwrite:
        flag = '-n'

    subprocess.call(['cp', flag, sourcepath, destpath])


# Argument parser
parser = argparse.ArgumentParser(
    description="A tool for adding external libraries to a Ubuntu SDK \
        application or scope.")

parser.add_argument(
    '-f',
    '--force-download',
    dest='force_download',
    action='store_true',
    help='force a new download of the packages')

parser.add_argument(
    '-d',
    '--delete-temp',
    dest='delete_temp',
    action='store_true',
    help='delete temp files at the end of the process')

parser.add_argument(
    '-e',
    '--extract-only',
    dest='extract_only',
    action='store_true',
    help='only create temp directory and extract the content of downloaded \
        packages')

parser.add_argument(
    '-c',
    '--custom-script',
    dest='script_path',
    type=str,
    help='run a custom script after the extraction of Debian packages and \
        before copying their content to the target destination. The tool will \
        execute the script with the path to the packages dump as argument. If \
        the \'-e\' flag has been specified, the script will be anyway \
        executed. This option is useful when you need to automatically modify \
        the content of temp folder (e.g. when you need to fix some path \
        before including the files in a click package).')

parser.add_argument(
    'manifest_path',
    type=str,
    help='path of json file containing the list of packages to be downloaded.')

parser.add_argument(
    'arch',
    type=str,
    choices=['amd64', 'i386', 'armhf'],
    help='CPU architecture ("amd64", "i386" or "armhf")')

parser.add_argument(
    'target_path',
    type=str,
    help='path to the target (a click package or a folder) where this \
        tool will include the downloaded binaries. If the folder does not \
        exist, it will be created.')

args = parser.parse_args()

# Variables
manifest_path = args.manifest_path
target_path = args.target_path
manifest_stat = os.stat(manifest_path)
temp_folder = os.path.join(
    '/tmp/',
    'tmp-click-deps-%s-%s-%s-%s' % (
        manifest_stat.st_dev,
        manifest_stat.st_ino,
        manifest_stat.st_size,
        manifest_stat.st_mtime))
temp_arch_folder = os.path.join(temp_folder, args.arch)
is_click_target = os.path.isfile(args.target_path)

# Check command line arguments
if not os.path.isfile(manifest_path):
    sys.exit("\n\nERROR: Package manifest is not a valid file. Exit...")

if not os.path.isfile(target_path) and target_path.endswith('.click'):
    sys.exit("\n\nERROR: The specified target .click does not exists.")
elif not os.path.isdir(target_path):
    print ("\n\nCreating dest folder\n{}".format(target_path))
    os.mkdir(target_path)

# Check if the script exist, if specified any.
if args.script_path and not os.path.exists(args.script_path):
    sys.exit("\n\nERROR: The specified script does not exists.")

# If -f argument has been specified, remove all the existing data before
# running this script.
if args.force_download:
    if os.path.isdir(temp_folder):
        print ("\nRemoving temp data of the previous run, as requested")
        delete_folder(temp_folder, True)

# Check if we already have run this script for the same target.
if not check_if_temp_folder_already_exists(temp_arch_folder):
    # Check internet connection
    if not check_internet_connection():
        sys.exit("\n\nERROR: An internet connection is required in order to \
            download packages from repositories.")

    # Create temp folder in /tmp
    print ("\nCreating temp folder in {}".format(temp_folder))
    os.mkdir(temp_folder)
    os.mkdir(temp_arch_folder)

    # Parse the JSON package list and get the download URL of the packages.
    debs_url_list = get_URLs_for_arch(
        manifest_path,
        args.arch,
        temp_folder)

    # Download packages from web
    for url in debs_url_list:
        download_file(url, temp_folder)

    # Extract DEBs packages
    print ("\nExtracting .deb packages to {}".format(temp_arch_folder))
    deb_pkgs_list = []
    for file in os.listdir(temp_folder):
        if file.endswith('.deb'):
            deb_pkgs_list.append(file)

    for deb_pkg in deb_pkgs_list:
        extract_deb_package(
            os.path.join(temp_folder, deb_pkg),
            temp_arch_folder)

    # If a script has been specified, run it.
    if args.script_path:
        print ("\nRunning the script at: {}\n".format(args.script_path))

        # Ensure that we can run the script, otherwise we don't have the
        # permission
        subprocess.call(['chmod', '+x', args.script_path])
        subprocess.call([
            args.script_path,
            temp_arch_folder,
            get_arch_triplet(args.arch)])
else:
    print ("\nPackages are already downloaded. Use them...")

# If -e or --extract-only flags have been specified, we have completed our work
if args.extract_only:
    print ("\n\nCompleted successfully.")
    sys.exit(0)

# Copy temp_arch folder content to its destination
if is_click_target:
    print ("\nExtracting target .click package")
    temp_click_folder = extract_click_package(
        args.target_path,
        os.path.join('/tmp/', str(get_timestamp())))

    print ("\nAdding extracted binaries to the package")
    copy_directory_content(temp_arch_folder, temp_click_folder)

    new_click_package_path = build_click_package(temp_click_folder)
    print ("\nCreated new .click package at:\n{}".format(new_click_package_path))

    print ("\nReplacing older .click package")
    copy_file(new_click_package_path, args.target_path, True)

    delete_folder(temp_click_folder, True)

else:
    print ("\nCopying extracted binaries to their destination")
    copy_directory_content(temp_arch_folder, args.target_path)

# Delete temp files
if args.delete_temp:
    print ("\)nRemoving temp files and directory, as requested")
    delete_folder(temp_folder, True)

# Exit
print ("\n\nCompleted successfully.")