~fginther/auto-package-testing/release-from-command-line

#! /usr/bin/python
""" Interface between britney and autopkgtest running on Jenkins

"""
# Copyright (C) 2012, Canonical Ltd (http://www.canonical.com/)
#
# Author: Jean-Baptiste Lallement <jean-baptiste.lallement@canonical.com>
#
# This software is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation, either version 3 of
# the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this software.  If not, see <http://www.gnu.org/licenses/>.
#
import apt_pkg
import os
import logging
import sys
from glob import glob
import subprocess
import yaml
import argparse
from adtjob import AdtJob
from aptcache import AptCache
from time import strftime
from shutil import copyfile
import json
import kombu

# 'all' is used to track the status across all archs.
ARCHS = ('i386', 'amd64', 'all')
DEFAULT_RELEASE = 'saucy'
DEFAULT_ARCH = 'amd64'
BINDIR = os.path.dirname(__file__)
# Do we run from a bzr checkout ?
if os.path.exists(os.path.join(BINDIR, "../.bzr")):
    sys.path.insert(1, BINDIR)

# Error codes
E_CONFIG = 1
E_APT = 2


class RecursiveDict(dict):
    """Implementation of perl's autovivification feature."""
    def __missing__(self, key):
        value = self[key] = type(self)()
        return value


class AdtBritney(object):
    """Interface between britney and autopkgtest running in Jenkins"""
    DEFAULT_RC = os.path.join(BINDIR, 'adt-britney.rc')

    config_defaults = {
        'arch': DEFAULT_ARCH,
        'release': DEFAULT_RELEASE,
        'aptroot': '/tmp/adt/apt',
        'datadir': '/tmp/adt/data',
        'apturi': 'http://archive.ubuntu.com/ubuntu/',
        'components': 'main restricted',
        'extrasources': '',
        'rsync_host': '',
        'amqp_host': '',
        'amqp_queue': '',
    }

    def __init__(self):
        """Constructor

        Parse command line argument and loads package list
        """
        self.cache = None
        self.pocket = ""
        self.args = None
        self.testsuites = []

        self.config = dict(self.config_defaults)
        self.__parse_args()
        self.__load_config()
        self.__init_apt()

        # Status Files
        datadir = os.path.abspath(
            os.path.expanduser(
                os.path.join(self.config['datadir'], 'adt',
                             self.config['release'] + self.pocket,
                             self.config['arch'])))
        if not os.path.exists(datadir):
            os.makedirs(datadir)
        elif not os.path.isdir(datadir):
            logging.error("'%s' is not a directory, exiting!", datadir)
            sys.exit(E_CONFIG)
        # release(-pocket)?_arch_package
        self.status_pattern = os.path.join(datadir, '%s%s_%s_%s')
        self.adt_cachedir = datadir
        self.adt_workdir = os.path.join(self.adt_cachedir, 'work')
        if not os.path.exists(self.adt_workdir):
            os.makedirs(self.adt_workdir)
        self.adt_archivedir = os.path.join(self.adt_cachedir, 'archive')
        if not os.path.exists(self.adt_archivedir):
            os.makedirs(self.adt_archivedir)

        # test runner AMQP queue
        if self.config.get('amqp_host'):
            self.amqp_con = kombu.Connection(self.config['amqp_host'])
            self.amqp_queue = self.amqp_con.SimpleQueue(self.config['amqp_queue'])
        else:
            self.amqp_con = None
            self.amqp_queue = None

        self.args.func()

    def __del__(self):
        if self.amqp_con:
            self.amqp_con.release()

    def __parse_args(self):
        """Parse command line arguments

        This method parses command line arguments and configures logging
        """

        parser = argparse.ArgumentParser(
            description='Check if a new version of a package or its '
            'dependencies is available from the archive and trigger an '
            'autopkgtest run if there is.')
        parser.add_argument('-d', '--debug', action='store_true',
                            default=False, help='enable debug mode')
        parser.add_argument('-c', '--config', default=self.DEFAULT_RC,
                            help='Path to configuration file (default=%s)' %
                            self.DEFAULT_RC)
        parser.add_argument('-a', '--arch', metavar='ARCH',
                            help='Architecture')
        parser.add_argument('-r', '--release', metavar='RELEASE',
                            help='Release')
        parser.add_argument('-P', '--use-proposed', action='store_true',
                            default=False,
                            help='Check if new versions are in -proposed and '
                            'enable -proposed in testbed.')
        parser.add_argument('-U', '--no-update', action='store_true',
                            default=False,
                            help='Do not update apt cache')
        subparsers = parser.add_subparsers()

        prequest = subparsers.add_parser(
            'request',
            help='Generates a request file or update it if passed in argument')
        prequest.add_argument(
            'requestfile', nargs='?',
            help='Request file. If no request file is speficied, it will be '
            'generated from the list of packages with an XS-Testsuite source '
            'header')
        prequest.add_argument('-O', '--output', default=None,
                              metavar='file',
                              help='Request file will be written to \'file\''
                              'instead of stdout')
        prequest.set_defaults(func=self.cmd_request)

        psubmit = subparsers.add_parser(
            'submit',
            help='Submit a request file to Jenkins and trigger the tests')
        psubmit.add_argument(
            'requestfile', help='Request file to submit to Jenkins')
        psubmit.set_defaults(func=self.cmd_submit)

        pcollect = subparsers.add_parser(
            'collect', help='Collect the results from the tests')
        pcollect.add_argument('-O', '--output', default=None,
                              metavar='file',
                              help='Request file will be written to \'file\''
                              'instead of stdout')
        pcollect.add_argument('-n', '--new-only', action='store_true',
                              default=False,
                              help='Display only new results otherwise the '
                              'history is displayed')

        pcollect.set_defaults(func=self.cmd_collect)

        pretry = subparsers.add_parser(
            'retry',
            help='Resubmit a test request for a list of packages')
        pretry.add_argument('sourcepackages', metavar='SRCPKG', nargs='+',
                            help='Name of the source package to retry')
        pretry.add_argument('-f', '--force', metavar='STATUS',
                            choices=['PASS', 'FAIL', 'RUNNING'],
                            help='Force the result to STATUS for a list of'
                            'source packages without considering the actual'
                            'result of the tests')
        pretry.set_defaults(func=self.cmd_retry)

        self.args = parser.parse_args()
        set_logging(self.args.debug)
        logging.debug(self.args)

    def __load_config(self):
        ''' Load a configuration file in yaml format

        :param path: path to configuration file '''
        path = self.args.config
        logging.debug('Loading configuration file: %s', path)
        path = os.path.abspath(os.path.expanduser(path))

        if not os.path.isfile(path):
            logging.warning('Configuration file doesn\'t exists (%s), using '
                            'default settings.', path)
            return

        try:
            cfg = yaml.load(file(path, 'r'))
        except yaml.scanner.ScannerError as exc:
            logging.error('Error loading configuration file, exiting!\n' +
                          '=' * 80 + '\n%s\n' + '=' * 80, exc)
            sys.exit(E_CONFIG)

        for kcfg in self.config:
            if kcfg in cfg:
                self.config[kcfg] = cfg[kcfg]

        if self.args.release:
            self.config['release'] = self.args.release
        if self.args.arch:
            self.config['arch'] = self.args.arch

    def __init_apt(self):
        ''' Initialize apt and create all required the files '''

        if not self.config['aptroot']:
            logging.error('No APTROOT defined, exiting!')
            sys.exit(E_CONFIG)
        # Create aptroot tree
        self.pocket = "-proposed" if self.args.use_proposed else ""

        self.config['aptroot'] = os.path.abspath(
            os.path.expanduser(self.config['aptroot']))

        if not os.path.isdir(self.config['aptroot']):
            logging.debug('Creating APT root directory (%s)',
                          self.config['aptroot'])
            os.makedirs(self.config['aptroot'])
        apt_pkg.init()
        apt_pkg.config.set('Dir', self.config['aptroot'])
        self.cache = AptCache(rootdir=self.config['aptroot'])

        # We don't need package indices in collect mode
        if self.args.func == self.cmd_collect:
            return

        if not self.args.no_update:
            self.cache.write_sources_list(self.config['apturi'],
                                          self.config['arch'],
                                          self.config['release'],
                                          self.config['components'],
                                          self.args.use_proposed)
            self.cache.update()
            self.cache.open()

        self.testsuites = self.get_packages_with_testsuite()
        logging.debug('Found %d packages with testsuite:\n%s',
                      len(self.testsuites), self.testsuites)

    def __get_release_header(self):
        '''Extract headers from release files'''
        from itertools import islice

        # Load header from Release files
        rfile_pattern = os.path.join(self.config['aptroot'],
                                     'var/lib/apt/lists/') +\
            '*_{}*_Release'.format(self.config['release'])

        header = []
        for rfilename in glob(rfile_pattern):
            if rfilename.endswith('{}-proposed_Release') and \
                    not self.args.use_proposed:
                continue
            with open(rfilename) as rfd:
                header += [x.strip() for x in list(islice(rfd, 6))
                           if x.startswith('Suite: ') or
                           x.startswith('Date: ')]
        return header

    def __write_result_file(self, lines, filename=None, header=True):
        '''Write packages to test to a request file or stdout if none is
        given
        '''
        # filename overrides self.args.output
        if filename is not None:
            outputfile = filename
        else:
            outputfile = self.args.output

        if not outputfile:
            logging.debug('Writing output to stdout')
            fdh = sys.stdout
        else:
            logging.debug('Writing output to %s', outputfile)
            fdh = open(outputfile, 'w')

        if header:
            header = self.__get_release_header()
            fdh.write('\n'.join(header) + '\n')

        for line in lines:
            fdh.write(line + '\n')

        if fdh != sys.stdout:
            fdh.close()

    def cmd_request(self):
        ''' Process or generate a request file '''
        logging.debug('Cmd request')

        lines = []
        reqlist = {}
        if self.args.requestfile:
            (header, reqlist) = load_request_file(self.args.requestfile)
        for pkgname in self.testsuites:
            causes = {}
            deps_path = self.status_pattern % (
                self.config['release'], self.pocket,
                self.config['arch'], pkgname)
            job = AdtJob(self.cache, deps_path,
                         use_proposed=self.args.use_proposed)
            if not job.package:
                job.release = self.config['release']
                job.pkgname = pkgname
                if (reqlist and not (pkgname in reqlist.keys() or
                                     job.run_required(reqlist))):
                    # New package but package or its dependencies are not
                    # in the list of requests
                    continue
                (pkgvers, pocket, pkgdeps) = job.get_package_info()
                logging.info('== New job')
                causes = {pkgname: pkgvers}
                status = 'NEW'
            else:
                if job.run_required(reqlist):
                    causes = job.causes
                    (pkgvers, pocket, pkgdeps) = job.get_package_info()
                    status = 'NEW'
            if causes:
                lines.append(
                    ' '.join((
                        pkgname, pkgvers, status,
                        ' '.join([
                            "%s %s" % (k, v) for k, v in causes.iteritems()
                        ]))))

        self.__write_result_file(lines)

    def cmd_submit(self, packages=None):
        ''' Submit a request a request file to jenkins '''
        logging.debug('Cmd submit')
        # Load the request file
        pkglist = {}
        if packages is None:
            packages = []
        if hasattr(self.args, 'requestfile'):
            (header, pkglist) = load_request_file(self.args.requestfile)

        for pkg in packages:
            if pkg not in pkglist:
                logging.debug("Adding '%s' to request list", pkg)
                pkglist[pkg] = {
                    'version': None,
                    'status': 'NEW',
                    'causes': {pkg: None}
                }

        for pkgname, pkgprops in pkglist.iteritems():
            deps_path = self.status_pattern % (
                self.config['release'], self.pocket,
                self.config['arch'], pkgname)
            job = AdtJob(self.cache, deps_path,
                         use_proposed=self.args.use_proposed)
            if ('causes' in pkgprops and
                    pkgname in pkgprops['causes'] and
                    pkgprops['causes'][pkgname] is None and job.version):
                pkgprops['causes'][pkgname] = job.version
            if not job.package:
                logging.info('== New job')
                job.release = self.config['release']
                job.pkgname = pkgname
                if 'causes' in pkgprops:
                    job.causes = pkgprops['causes']
                ret = job.submit(os.path.join(self.config['rsync_host'],
                                              self.config['release'], 'in/'),
                                 force=True)
            else:
                job.status = 'NEW'
                if 'causes' in pkgprops:
                    job.causes = pkgprops['causes']
                force = self.args.func == self.cmd_retry
                ret = job.submit(os.path.join(self.config['rsync_host'],
                                              self.config['release'], 'in/'),
                                 force=force)

            if ret:
                # Copy to state file to workdir
                dest = os.path.join(self.adt_workdir,
                                    os.path.basename(deps_path) + '.' +
                                    strftime('%Y%m%d-%H%M%S') + '.state')
                copyfile(deps_path, dest)

            # also submit the job to the UCI airline through AMQP; FIXME: this is
            # temporary to run airline testing side by side with Jenkins testing
            # until we switch over
            # FIXME: track architectures on a higher level; for now, just hardcode
            # them
            if self.amqp_queue:
                for arch in ['i386', 'amd64']:
                    msg = json.dumps({'series': self.config['release'],
                                      'architecture': arch,
                                      'package': job.pkgname})
                    logging.debug('submitting AMQP request: %s', msg)
                    self.amqp_queue.put(msg)

        if (len(pkglist) > 0 and hasattr(self.args, 'requestfile')):
            self.__copy_to_archive(self.args.requestfile)

    def __load_results(self, filelist):
        """
        Loads result files

        This method loads the results from a list of result files (*.result in
        working directory) and returns a dictionary container the results.
        The result file which all the architecture have been tested are
        archived. Those that have not the archs covered are kept to update the
        history later on or allow test retry without having a state file
        matching a request from britney

        Args:
            filelist: List of files to process

        Returns:
            Dictionary containing the results that have been loaded
        """
        logging.debug("Loading result files")
        # Load all the results in memory
        test_results = RecursiveDict()
        result_files = RecursiveDict()

        for result_file in filelist:
            try:
                (rel, arc, pkg, ver, res, dep) = load_result(result_file)
            except ValueError:
                logging.warning('Invalid file: %s', result_file)
                self.__move_to_archive(result_file)
                continue

            if not pkg in test_results[rel][arc]:
                test_results[rel][arc][pkg] = []
            if not (res, dep) in test_results[rel][arc][pkg]:
                test_results[rel][arc][pkg].append((res, dep))

            # Build a list of test result files where pkg and arc are inverted
            # compared to results. The goal is to identify the packages for
            # which we have all the result files and can archive the results.
            if not arc in result_files[rel][pkg]:
                result_files[rel][pkg][arc] = []
            result_files[rel][pkg][arc].append(result_file)

        # Only archive files where all the archs have been tested
        for rel, pkgs in result_files.iteritems():
            for pkg, arcs in pkgs.iteritems():
                if len(arcs) >= len(ARCHS) - 1:
                    for arc, files in arcs.iteritems():
                        for resfile in files:
                            self.__move_to_archive(resfile)
        return test_results

    def __process_state_files(self, filelist, results):
        """
        Update state files from test results

        Process a list of state files from a list of test results.

        Args:
            filelist: List of state files
            results: Dictionary containing the test results

        Returns:
            List of results that have been processed
        """
        all_results = []
        # Loads the state files and compare to the results
        for state_file in filelist:
            logging.debug('Processing %s', state_file)
            state = json.load(file(state_file))

            # File already processed move it away
            if state['status']['all'] != 'RUNNING':
                logging.debug('Already done, moving %s', state_file)
                self.__move_to_archive(state_file)
                continue

            global_result = True
            accepted_arch = 0
            all_done = False
            for arch in ARCHS:
                if arch == 'all':
                    continue

                # Already collected a result for this file, moving on
                if state['status'][arch] != 'RUNNING':
                    accepted_arch += 1
                    global_result = global_result & (
                        state['status'][arch] == 'PASS')
                    continue
                try:
                    results_for_arch = results[
                        state['release']][arch][state['package']]
                except KeyError:
                    # No result for package
                    continue

                # Check which result file matches the test
                # Stop on the first with packages in causes and a version >=
                # the version that triggered the test
                for (test_result, test_depends) in results_for_arch:
                    logging.debug('result %s : %s', test_result, test_depends)
                    accepted_causes = 0
                    for cause_pkg, cause_version \
                            in state['causes'].iteritems():
                        logging.debug('cause: %s %s', cause_pkg, cause_version)
                        if not cause_pkg in test_depends:
                            logging.debug('rejected: not in test depends: %s',
                                          cause_pkg)
                            break
                        test_version = test_depends[cause_pkg]
                        # Version tested < Version requested => reject result
                        if apt_pkg.version_compare(test_version,
                                                   cause_version) < 0:
                            logging.debug('rejected: test version <'
                                          'pkg version')
                            break
                        accepted_causes += 1
                        logging.debug('accepted for cause %s %s', cause_pkg,
                                      cause_version)

                    # Exit when a result is found that matches all the criteria
                    if accepted_causes == len(state['causes']):
                        accepted_arch += 1
                        logging.debug('accepted for arch: %s %s %s',
                                      arch, test_result, test_depends)
                        state['status'][arch] = test_result
                        global_result = global_result & (test_result == 'PASS')
                        break

            if accepted_arch == len(ARCHS) - 1:  # Removes 'all'
                state['status']['all'] = 'PASS' if global_result else 'FAIL'
                all_done = True
            all_results.append(state)
            # Update state file
            with open(state_file, 'w') as fp:
                logging.debug("Writing status file")
                json.dump(state, fp)
            logging.debug('TEST RESULT: %s', global_result)
            if all_done:
                self.__move_to_archive(state_file)

        logging.debug('Runs accepted: %d', len(all_results))
        return all_results

    def __update_history(self, history, results):
        """
        Update history with results even without state files

        This method updates the history file used by britney to report test
        results, even if there is no state file. Tihs allows to replay tests
        directly from jenkins without an explicit request from britney. Tihs
        is useful when there is an infrastructure failure for example.

        We read all the history and for all tests marked as FAIL or RUNNING,
        check if a newer version of the package has been tested and is
        successful if the result was FAIL; or PASS or FAIL if the result was
        RUNNING

        Args:
            history: Dictionary containing the historical results
            results: New results loaded from result files. Only results with
                all the archs will be processed

        Returns:
            An updated history

        """

        def result_is_acceptable(history, tested):
            """
            Compare package/version in history to version of the packages that
            have been tested and return True if packages tested have a version
            >= versions in history

            Args:
                history: History record
                tested: Tuple container result of the test and packages that
                    have been tested
            Returns:
                True if packages that have been tested are newer or equal to
                the version requested for the test
            """
            pkgs = history['causes']
            pkgs[history['package']] = history['version']

            for pkg in pkgs:
                if pkg not in tested[1]:
                    continue  # Not a dependency anymore
                if apt_pkg.version_compare(pkgs[pkg], tested[1][pkg]) > 0:
                    # Older version that requested -> discard
                    return False
            return True

        logging.debug("Reprocessing history")
        linenum = -1
        for hrow in history:
            linenum += 1
            if hrow['status']['all'] == 'PASS':
                continue

            # Check if all the results are there for this result/package
            testres = 'UNK'
            # Flag that indicates if test results are available for all archs
            arch_seen = 0
            resrow = None
            for arch in ARCHS:
                if arch == 'all':
                    continue
                # We need results for all the archs
                if hrow['package'] not in results[
                        self.config['release']][arch]:
                    break
                resrow = results[
                    self.config['release']][arch][hrow['package']][-1]
                # Check if the package and causes have a version >= run in
                # history
                if not result_is_acceptable(hrow, resrow):
                    break

                if resrow[0] == 'FAIL':
                    testres = 'FAIL'
                elif resrow[0] == 'PASS' and testres != 'FAIL':
                    testres = 'PASS'
                arch_seen += 1

            # There is one arch missing or , skip result
            if arch_seen < len(ARCHS) - 1 or testres == 'UNK':
                continue

            # Update history with the latest result and versions
            if testres != 'UNK':
                history[linenum]['status']['all'] = testres
            if history[linenum]['package'] in resrow[1]:
                history[linenum]['version'] = resrow[1][
                    history[linenum]['package']]
            else:
                logging.debug("package %s not found in tested packages",
                              history[linenum]['package'])

            for pkg in history[linenum]['causes']:
                if pkg in resrow[1]:
                    history[linenum]['causes'][pkg] = resrow[1][pkg]
                else:
                    logging.debug("cause %s not found in tested packages", pkg)
        return history

    def cmd_collect(self):
        ''' Collect results from jenkins '''
        logging.debug('Cmd collect')

        state_pattern = os.path.join(
            self.adt_workdir, self.config['release'] + '*.state')
        result_pattern = os.path.join(
            self.adt_workdir, self.config['release'] + '*.result')

        if not self.__sync_results():
            return False

        result_files = sorted(glob(result_pattern))
        state_files = sorted(glob(state_pattern))

        # Load all the results in memory
        test_results = self.__load_results(result_files)
        all_results = self.__process_state_files(state_files, test_results)

        # Update history file
        history_path = os.path.join(self.adt_workdir, "results.history")
        logging.debug('Loading history from %s', history_path)
        history_res = load_results_history(history_path)
        history_all = merge_results(history_res, all_results)
        history_all = self.__update_history(history_all, test_results)

        def resultdict2csv(results):
            lines = []
            for rec in results:
                lines.append(' '.join([rec['package'], rec['version'],
                                       rec['status']['all']] +
                                      [" ".join(x)
                                       for x in sorted(
                                           rec['causes'].iteritems())]))
            return lines
        lines = deduplicate_and_sort(resultdict2csv(history_all))

        self.__write_result_file(lines, header=False, filename=history_path)

        # Write new results
        if self.args.new_only:
            lines = resultdict2csv(all_results)
            self.__write_result_file(lines, header=False)
        else:
            # Full history print lines again
            self.__write_result_file(lines, header=False)

    def cmd_retry(self):
        """"
        Submit a test request for a list of package passed in argument

        Args:
            None
        Returns:
            None
        """
        if self.args.force is not None:
            # Read all the history and change the status of all the lines
            # corresponding to the packages on the cmd line
            history_path = os.path.join(self.adt_workdir, "results.history")
            lines = []
            with open(history_path, 'r') as fhist:
                for line in fhist:
                    linebits = line.strip().split()
                    for pkg in self.args.sourcepackages:
                        if pkg == linebits[0]:
                            logging.debug("Changing status of '%s' to '%s'",
                                          pkg, self.args.force)
                            linebits[2] = self.args.force
                    lines.append(" ".join(linebits))
            self.__write_result_file(lines, header=False,
                                     filename=history_path)
            return True
        else:
            self.cmd_submit(self.args.sourcepackages)

    def __move_to_archive(self, source):
        '''Copy a file to the archive folder, move it if move is True'''
        self.__copy_to_archive(source, move=True)

    def __copy_to_archive(self, source, move=False):
        '''Copy a file to the archive folder, move it if move is True'''
        num = 0
        dest_dir = os.path.join(self.adt_archivedir, strftime('%Y/%m/%d'))
        dest = os.path.join(dest_dir, os.path.basename(source))
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir)
        tmpdest = dest
        while os.path.exists(tmpdest):
            num += 1
            tmpdest = "{}.{}".format(dest, num)
        dest = tmpdest
        if move:
            logging.debug('Moving %s -> %s', source, dest)
            os.rename(source, dest)
        else:
            logging.debug('Copying %s -> %s', source, dest)
            copyfile(source, dest)

    def __sync_results(self):
        ''' Update the latest results from remote host'''
        logging.debug('Syncing latest results')
        remote = os.path.join(self.config['rsync_host'],
                              self.config['release'], 'out/')
        cmd = ['rsync', '-a', '--remove-source-files', remote,
               self.adt_workdir]
        logging.debug('cmd: %s', cmd)
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as exc:
            logging.error('Command failed: %s', exc)
            return False
        return True

    def get_packages_with_testsuite(self):
        """Build list of packages with an XS-Testsuite header

        This method uses grep-dctrl to find packages with a
        'Testsuite: autopkgtest' source header from the Sources indexes
        """

        sources_pattern = "*{release}*_*source_Sources".format(
            release=self.config['release'])
        sources_path = os.path.join(
            self.config['aptroot'], 'var/lib/apt/lists/', sources_pattern)
        logging.debug("Searching for sources indices in '%s'", sources_path)
        sources = glob(sources_path)

        if not sources:
            logging.error('No sources file found, exiting!')
            sys.exit(E_APT)
        grepdctrl_cmd = ['grep-dctrl', '-FTestsuite', '-sPackage',
                         '-n', 'autopkgtest']
        grepdctrl_cmd += sources
        logging.debug('Executing cmd: %s', " ".join(grepdctrl_cmd))

        try:
            packages = subprocess.check_output(grepdctrl_cmd).strip()
        except subprocess.CalledProcessError, e:
            logging.error('Failed to execute \'%s\'\n%s',
                          " ".join(grepdctrl_cmd), e)
            return False

        pkgs = sorted(set([x for x in packages.split('\n') if x]))
        return pkgs


def load_result(path):
    ''' Load a result file '''
    depends = []
    with open(path, 'r') as f:
        data = f.readline().split()
    (release, arch, pkgname, pkgvers, result) = [data.pop(0) for x in
                                                 data[:5]]
    i = iter(data)
    depends = dict(zip(i, i))
    depends[pkgname] = pkgvers

    return (release, arch, pkgname, pkgvers, result, depends)


def load_results_history(path):
    """ Load history of results from path
    """
    depends = []
    if not os.path.isfile(path):
        logging.warning('File doesn\'t exists: %s', path)
        return []
    with open(path, 'r') as f:
        lines = f.readlines()

    results = []
    for data in lines:
        data = data.split()
        (pkgname, pkgvers, result) = [data.pop(0) for x in data[:3]]
        i = iter(data)
        depends = dict(zip(i, i))
        results.append({"package": pkgname,
                        "version": pkgvers,
                        "status": {"all": result},
                        "causes": depends,
                        "depends": {}})

    return results


def load_request_file(filename):
    """Loads a request file with format
    <package version> <status> [<package version> ...]
    """

    reqfile = filename
    pkglist = {}

    if not reqfile:
        return pkglist

    if not os.path.isfile(filename):
        logging.error('File doesn\'t exists (%s), exiting!', filename)
        sys.exit(E_CONFIG)

    with open(filename, 'r') as fdh:
        content = fdh.readlines()

    header = []
    for line in content:
        # Header
        if line.startswith('Suite:') or line.startswith('Date:'):
            header.append(line.strip())
            continue

        # Packages
        linebits = line.strip().split()
        if len(linebits) < 2:
            logging.warning('Invalid line format: %s, skipped',
                            line.strip())
            continue
        pkgname = linebits.pop(0)
        pkgvers = linebits.pop(0)
        pkglist[pkgname] = {
            'version': pkgvers,
            'status': 'NEW',
            'causes': {}
        }

        try:
            pkglist[pkgname]['status'] = linebits.pop(0).upper()
            while True:
                cname = linebits.pop(0)
                cvers = linebits.pop(0)
                pkglist[pkgname]['causes'][cname] = cvers
        except IndexError:
            # End of the list
            pass

    logging.debug('Loaded request file:\n%s', pkglist)

    return (header, pkglist)


def merge_results(results_orig, results_new):
    """ Returns history+new result and merges the results when packages and
    causes of the test are identical and versions are greater or equal
    """
    flattened = {}
    for result in results_orig:
        # Create a key made of the package names to make finding same set of
        # package/causes easier
        key = result['package'] + "".join(result['causes'].keys())
        flattened[key] = result
    for newres in results_new:
        key = newres['package'] + "".join(newres['causes'].keys())
        if key in flattened:
            # Check if versions of package and causes are equal or higher
            # than previous run
            if (apt_pkg.version_compare(
                    flattened[key]['version'], newres['version']) <= 0):
                for pkg, vers in flattened[key]['causes'].iteritems():
                    alldepsge = True
                    if not (pkg in newres['causes'] and
                            apt_pkg.version_compare(
                                vers, newres['causes'][pkg]) <= 0):
                        alldepsge = False
                    if alldepsge:
                        # Replace previous result
                        flattened[key] = newres
        else:
            # New package/deps
            flattened[key] = newres
    return flattened.values()


def deduplicate_and_sort(history_lines):
    """ Remove duplicate, split list of dependency and sort by package name and
    version
    """
    results = RecursiveDict()
    lines = []
    for line in history_lines:
        line = line.split()
        try:
            (pname, pversion, result) = line[:3]
            i = iter(line[3:])
            causes = dict(zip(i, i))
        except ValueError:
            print('E: Invalid record: %s' % line)
            continue
        for cname, cversion in causes.iteritems():
            if not results[pname][pversion][cname][cversion]:
                results[pname][pversion][cname][cversion] = result
            else:
                if results[pname][pversion][cname][cversion] == 'PASS':
                    # If it passed once it cannot fail with the same version
                    continue
                elif result == 'PASS':
                    # If it failed then pass, accept the pass. Failure might
                    # be caused by another dependency or an infra error
                    results[pname][pversion][cname][cversion] = 'PASS'

    for pname in sorted(results):
        pversions = sorted(results[pname].keys(), cmp=apt_pkg.version_compare)
        for pversion in pversions:
            for cname in sorted(results[pname][pversion]):
                for cversion in sorted(results[pname][pversion][cname].keys(),
                                       cmp=apt_pkg.version_compare):
                    lines.append(" ".join(
                        [pname, pversion,
                         results[pname][pversion][cname][cversion], cname,
                         cversion]))
    return lines


def set_logging(debugmode=False):
    """Initialize logging"""
    logging.basicConfig(
        level=logging.DEBUG if debugmode else logging.INFO,
        format="%(asctime)s %(levelname)s %(message)s")
    logging.debug('Debug mode enabled')


if __name__ == "__main__":
    AdtBritney()