~launchpad-pqm/launchpad/devel

# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""The code import worker. This imports code from foreign repositories."""

__metaclass__ = type

__all__ = [

    'BazaarBranchStore',

    'BzrImportWorker',

    'BzrSvnImportWorker',

    'CSCVSImportWorker',

    'CodeImportBranchOpenPolicy',

    'CodeImportSourceDetails',

    'CodeImportWorkerExitCode',

    'ForeignTreeStore',

    'GitImportWorker',

    'ImportWorker',

    'ToBzrImportWorker',

    'get_default_bazaar_branch_store',

    ]

import io

import os

import shutil

import subprocess

from urlparse import (

    urlsplit,

    urlunsplit,

    )

# FIRST Ensure correct plugins are loaded. Do not delete this comment or the

# line below this comment.

import lp.codehosting

from bzrlib.branch import (

    Branch,

    InterBranch,

    )

from bzrlib.bzrdir import (

    BzrDir,

    BzrDirFormat,

    )

from bzrlib.errors import (

    ConnectionError,

    InvalidEntryName,

    NoRepositoryPresent,

    NoSuchFile,

    NotBranchError,

    TooManyRedirections,

    )

from bzrlib.transport import (

    get_transport_from_path,

    get_transport_from_url,

    )

import bzrlib.ui

from bzrlib.upgrade import upgrade

from bzrlib.urlutils import (

    join as urljoin,

    local_path_from_url,

    )

import cscvs

from cscvs.cmds import totla

import CVS

from dulwich.errors import GitProtocolError

from dulwich.protocol import (

    pkt_line,

    Protocol,

    )

from lazr.uri import (

    InvalidURIError,

    URI,

    )

from pymacaroons import Macaroon

import SCM

from zope.component import getUtility

from zope.security.proxy import removeSecurityProxy

from lp.code.enums import RevisionControlSystems

from lp.code.interfaces.branch import (

    get_blacklisted_hostnames,

    IBranch,

    )

from lp.code.interfaces.codehosting import (

    branch_id_alias,

    compose_public_url,

    )

from lp.codehosting.codeimport.foreigntree import CVSWorkingTree

from lp.codehosting.codeimport.tarball import (

    create_tarball,

    extract_tarball,

    )

from lp.codehosting.codeimport.uifactory import LoggingUIFactory

from lp.codehosting.safe_open import (

    BadUrl,

    BranchOpenPolicy,

    SafeBranchOpener,

    )

from lp.services.config import config

from lp.services.macaroons.interfaces import IMacaroonIssuer

from lp.services.propertycache import cachedproperty

from lp.services.timeout import urlfetch

from lp.services.utils import sanitise_urls

class CodeImportBranchOpenPolicy(BranchOpenPolicy):

    """Branch open policy for code imports.

    In summary:

     - follow references,

     - only open non-Launchpad URLs for imports from Bazaar to Bazaar or

       from Git to Git

     - only open the allowed schemes

    """

    allowed_schemes = ['http', 'https', 'svn', 'git', 'ftp', 'bzr']

    def __init__(self, rcstype, target_rcstype):

        self.rcstype = rcstype

        self.target_rcstype = target_rcstype

    def shouldFollowReferences(self):

        """See `BranchOpenPolicy.shouldFollowReferences`.

        We traverse branch references for MIRRORED branches because they

        provide a useful redirection mechanism and we want to be consistent

        with the bzr command line.

        """

        return True

    def transformFallbackLocation(self, branch, url):

        """See `BranchOpenPolicy.transformFallbackLocation`.

        For mirrored branches, we stack on whatever the remote branch claims

        to stack on, but this URL still needs to be checked.

        """

        return urljoin(branch.base, url), True

    def checkOneURL(self, url):

        """See `BranchOpenPolicy.checkOneURL`.

        We refuse to mirror Bazaar branches from Launchpad, or any branches

        from a ssh-like or file URL.

        """

        try:

            uri = URI(url)

        except InvalidURIError:

            raise BadUrl(url)

        if self.rcstype == self.target_rcstype:

            launchpad_domain = config.vhost.mainsite.hostname

            if uri.underDomain(launchpad_domain):

                raise BadUrl(url)

        for hostname in get_blacklisted_hostnames():

            if uri.underDomain(hostname):

                raise BadUrl(url)

        if uri.scheme not in self.allowed_schemes:

            raise BadUrl(url)

class CodeImportWorkerExitCode:

    """Exit codes used by the code import worker script."""

    SUCCESS = 0

    FAILURE = 1

    SUCCESS_NOCHANGE = 2

    SUCCESS_PARTIAL = 3

    FAILURE_INVALID = 4

    FAILURE_UNSUPPORTED_FEATURE = 5

    FAILURE_FORBIDDEN = 6

    FAILURE_REMOTE_BROKEN = 7

class BazaarBranchStore:

    """A place where Bazaar branches of code imports are kept."""

    def __init__(self, transport):

        """Construct a Bazaar branch store based at `transport`."""

        self.transport = transport

    def _getMirrorURL(self, db_branch_id):

        """Return the URL that `db_branch` is stored at."""

        return urljoin(self.transport.base, '%08x' % db_branch_id)

    def pull(self, db_branch_id, target_path, required_format,

             needs_tree=False, stacked_on_url=None):

        """Pull down the Bazaar branch of an import to `target_path`.

        :return: A Bazaar branch for the code import corresponding to the

            database branch with id `db_branch_id`.

        """

        remote_url = self._getMirrorURL(db_branch_id)

        try:

            remote_bzr_dir = BzrDir.open(remote_url)

        except NotBranchError:

            local_branch = BzrDir.create_branch_and_repo(

                target_path, format=required_format)

            if needs_tree:

                local_branch.bzrdir.create_workingtree()

            if stacked_on_url:

                local_branch.set_stacked_on_url(stacked_on_url)

            return local_branch

        # The proper thing to do here would be to call

        # "remote_bzr_dir.sprout()".  But 2a fetch slowly checks which

        # revisions are in the ancestry of the tip of the remote branch, which

        # we strictly don't care about, so we just copy the whole thing down

        # at the vfs level.

        control_dir = remote_bzr_dir.root_transport.relpath(

            remote_bzr_dir.transport.abspath('.'))

        target = get_transport_from_path(target_path)

        target_control = target.clone(control_dir)

        target_control.create_prefix()

        remote_bzr_dir.transport.copy_tree_to_transport(target_control)

        local_bzr_dir = BzrDir.open_from_transport(target)

        if local_bzr_dir.needs_format_conversion(format=required_format):

            try:

                local_bzr_dir.root_transport.delete_tree('backup.bzr')

            except NoSuchFile:

                pass

            upgrade(target_path, required_format, clean_up=True)

        if needs_tree:

            local_bzr_dir.create_workingtree()

        return local_bzr_dir.open_branch()

    def push(self, db_branch_id, bzr_branch, required_format,

             stacked_on_url=None):

        """Push up `bzr_branch` as the Bazaar branch for `code_import`.

        :return: A boolean that is true if the push was non-trivial

            (i.e. actually transferred revisions).

        """

        self.transport.create_prefix()

        target_url = self._getMirrorURL(db_branch_id)

        try:

            remote_branch = Branch.open(target_url)

        except NotBranchError:

            remote_branch = BzrDir.create_branch_and_repo(

                target_url, format=required_format)

            old_branch = None

        else:

            if remote_branch.bzrdir.needs_format_conversion(

                    required_format):

                # For upgrades, push to a new branch in

                # the new format. When done pushing,

                # retire the old .bzr directory and rename

                # the new one in place.

                old_branch = remote_branch

                upgrade_url = urljoin(target_url, "backup.bzr")

                try:

                    remote_branch.bzrdir.root_transport.delete_tree(

                        'backup.bzr')

                except NoSuchFile:

                    pass

                remote_branch = BzrDir.create_branch_and_repo(

                    upgrade_url, format=required_format)

            else:

                old_branch = None

        # This can be done safely, since only modern formats are used to

        # import to.

        if stacked_on_url is not None:

            remote_branch.set_stacked_on_url(stacked_on_url)

        pull_result = remote_branch.pull(bzr_branch, overwrite=True)

        # Because of the way we do incremental imports, there may be revisions

        # in the branch's repo that are not in the ancestry of the branch tip.

        # We need to transfer them too.

        remote_branch.repository.fetch(bzr_branch.repository)

        if old_branch is not None:

            # The format has changed; move the new format

            # branch in place.

            base_transport = old_branch.bzrdir.root_transport

            base_transport.delete_tree('.bzr')

            base_transport.rename("backup.bzr/.bzr", ".bzr")

            base_transport.rmdir("backup.bzr")

        return pull_result.old_revid != pull_result.new_revid

def get_default_bazaar_branch_store():

    """Return the default `BazaarBranchStore`."""

    return BazaarBranchStore(

        get_transport_from_url(config.codeimport.bazaar_branch_store))

class CodeImportSourceDetails:

    """The information needed to process an import.

    As the worker doesn't talk to the database, we don't use

    `CodeImport` objects for this.

    The 'fromArguments' and 'asArguments' methods convert to and from a form

    of the information suitable for passing around on executables' command

    lines.

    :ivar target_id: The id of the Bazaar branch or the path of the Git

        repository associated with this code import, used for locating the

        existing import and the foreign tree.

    :ivar rcstype: 'cvs', 'git', 'bzr-svn', 'bzr' as appropriate.

    :ivar target_rcstype: 'bzr' or 'git' as appropriate.

    :ivar url: The branch URL if rcstype in ['bzr-svn', 'git', 'bzr'], None

        otherwise.

    :ivar cvs_root: The $CVSROOT if rcstype == 'cvs', None otherwise.

    :ivar cvs_module: The CVS module if rcstype == 'cvs', None otherwise.

    :ivar stacked_on_url: The URL of the branch that the associated branch

        is stacked on, if any.

    :ivar macaroon: A macaroon granting authority to push to the target

        repository if target_rcstype == 'git', None otherwise.

    """

    def __init__(self, target_id, rcstype, target_rcstype, url=None,

                 cvs_root=None, cvs_module=None, stacked_on_url=None,

                 macaroon=None):

        self.target_id = target_id

        self.rcstype = rcstype

        self.target_rcstype = target_rcstype

        self.url = url

        self.cvs_root = cvs_root

        self.cvs_module = cvs_module

        self.stacked_on_url = stacked_on_url

        self.macaroon = macaroon

    @classmethod

    def fromArguments(cls, arguments):

        """Convert command line-style arguments to an instance."""

        target_id = arguments.pop(0)

        rcstype = arguments.pop(0)

        # XXX cjwatson 2016-10-12: Remove compatibility code once the

        # scheduler always passes both source and target types.

        if ':' in rcstype:

            rcstype, target_rcstype = rcstype.split(':', 1)

        else:

            target_rcstype = 'bzr'

        if rcstype in ['bzr-svn', 'git', 'bzr']:

            url = arguments.pop(0)

            if target_rcstype == 'bzr':

                try:

                    stacked_on_url = arguments.pop(0)

                except IndexError:

                    stacked_on_url = None

            else:

                stacked_on_url = None

            cvs_root = cvs_module = None

        elif rcstype == 'cvs':

            url = None

            stacked_on_url = None

            [cvs_root, cvs_module] = arguments

        else:

            raise AssertionError("Unknown rcstype %r." % rcstype)

        if target_rcstype == 'bzr':

            target_id = int(target_id)

            macaroon = None

        elif target_rcstype == 'git':

            macaroon = Macaroon.deserialize(arguments.pop(0))

        else:

            raise AssertionError("Unknown target_rcstype %r." % target_rcstype)

        return cls(

            target_id, rcstype, target_rcstype, url, cvs_root, cvs_module,

            stacked_on_url, macaroon)

    @classmethod

    def fromCodeImportJob(cls, job):

        """Convert a `CodeImportJob` to an instance."""

        code_import = job.code_import

        target = code_import.target

        if IBranch.providedBy(target):

            if target.stacked_on is not None and not target.stacked_on.private:

                stacked_path = branch_id_alias(target.stacked_on)

                stacked_on_url = compose_public_url('http', stacked_path)

            else:

                stacked_on_url = None

            target_id = target.id

        else:

            # We don't have a better way to identify the target repository

            # than the mutable unique name, but the macaroon constrains

            # pushes tightly enough that the worst case is an authentication

            # failure.

            target_id = target.unique_name

        if code_import.rcs_type == RevisionControlSystems.BZR_SVN:

            return cls(

                target_id, 'bzr-svn', 'bzr', str(code_import.url),

                stacked_on_url=stacked_on_url)

        elif code_import.rcs_type == RevisionControlSystems.CVS:

            return cls(

                target_id, 'cvs', 'bzr',

                cvs_root=str(code_import.cvs_root),

                cvs_module=str(code_import.cvs_module))

        elif code_import.rcs_type == RevisionControlSystems.GIT:

            if IBranch.providedBy(target):

                return cls(

                    target_id, 'git', 'bzr', str(code_import.url),

                    stacked_on_url=stacked_on_url)

            else:

                issuer = getUtility(IMacaroonIssuer, 'code-import-job')

                macaroon = removeSecurityProxy(issuer).issueMacaroon(job)

                return cls(

                    target_id, 'git', 'git', str(code_import.url),

                    macaroon=macaroon)

        elif code_import.rcs_type == RevisionControlSystems.BZR:

            return cls(

                target_id, 'bzr', 'bzr', str(code_import.url),

                stacked_on_url=stacked_on_url)

        else:

            raise AssertionError("Unknown rcstype %r." % code_import.rcs_type)

    def asArguments(self):

        """Return a list of arguments suitable for passing to a child process.

        """

        result = [

            str(self.target_id), '%s:%s' % (self.rcstype, self.target_rcstype)]

        if self.rcstype in ['bzr-svn', 'git', 'bzr']:

            result.append(self.url)

            if self.stacked_on_url is not None:

                result.append(self.stacked_on_url)

        elif self.rcstype == 'cvs':

            result.append(self.cvs_root)

            result.append(self.cvs_module)

        else:

            raise AssertionError("Unknown rcstype %r." % self.rcstype)

        if self.target_rcstype == 'git':

            # XXX cjwatson 2016-10-12: Consider arranging for this to be

            # passed to worker processes in the environment instead.

            result.append(self.macaroon.serialize())

        return result

class ImportDataStore:

    """A store for data associated with an import.

    Import workers can store and retreive files into and from the store using

    `put()` and `fetch()`.

    So this store can find files stored by previous versions of this code, the

    files are stored at ``<BRANCH ID IN HEX>.<EXT>`` where BRANCH ID comes

    from the CodeImportSourceDetails used to construct the instance and EXT

    comes from the local name passed to `put` or `fetch`.

    """

    def __init__(self, transport, source_details):

        """Initialize an `ImportDataStore`.

        :param transport: The transport files will be stored on.

        :param source_details: The `CodeImportSourceDetails` object, used to

            know where to store files on the remote transport.

        """

        self.source_details = source_details

        self._transport = transport

        self._target_id = source_details.target_id

    def _getRemoteName(self, local_name):

        """Convert `local_name` to the name used to store a file.

        The algorithm is a little stupid for historical reasons: we chop off

        the extension and stick that on the end of the branch id from the

        source_details we were constructed with, in hex padded to 8

        characters.  For example 'tree.tar.gz' might become '0000a23d.tar.gz'

        or 'git.db' might become '00003e4.db'.

        :param local_name: The local name of the file to be stored.

        :return: The name to store the file as on the remote transport.

        """

        if '/' in local_name:

            raise AssertionError("local_name must be a name, not a path")

        dot_index = local_name.index('.')

        if dot_index < 0:

            raise AssertionError("local_name must have an extension.")

        ext = local_name[dot_index:]

        return '%08x%s' % (self._target_id, ext)

    def fetch(self, filename, dest_transport=None):

        """Retrieve `filename` from the store.

        :param filename: The name of the file to retrieve (must be a filename,

            not a path).

        :param dest_transport: The transport to retrieve the file to,

            defaulting to ``get_transport_from_path('.')``.

        :return: A boolean, true if the file was found and retrieved, false

            otherwise.

        """

        if dest_transport is None:

            dest_transport = get_transport_from_path('.')

        remote_name = self._getRemoteName(filename)

        if self._transport.has(remote_name):

            dest_transport.put_file(

                filename, self._transport.get(remote_name))

            return True

        else:

            return False

    def put(self, filename, source_transport=None):

        """Put `filename` into the store.

        :param filename: The name of the file to store (must be a filename,

            not a path).

        :param source_transport: The transport to look for the file on,

            defaulting to ``get_transport('.')``.

        """

        if source_transport is None:

            source_transport = get_transport_from_path('.')

        remote_name = self._getRemoteName(filename)

        local_file = source_transport.get(filename)

        self._transport.create_prefix()

        try:

            self._transport.put_file(remote_name, local_file)

        finally:

            local_file.close()

class ForeignTreeStore:

    """Manages retrieving and storing foreign working trees.

    The code import system stores tarballs of CVS and SVN working trees on

    another system. The tarballs are kept in predictable locations based on

    the ID of the branch associated to the `CodeImport`.

    The tarballs are all kept in one directory. The filename of a tarball is

    XXXXXXXX.tar.gz, where 'XXXXXXXX' is the ID of the `CodeImport`'s branch

    in hex.

    """

    def __init__(self, import_data_store):

        """Construct a `ForeignTreeStore`.

        :param transport: A writable transport that points to the base

            directory where the tarballs are stored.

        :ptype transport: `bzrlib.transport.Transport`.

        """

        self.import_data_store = import_data_store

    def _getForeignTree(self, target_path):

        """Return a foreign tree object for `target_path`."""

        source_details = self.import_data_store.source_details

        if source_details.rcstype == 'cvs':

            return CVSWorkingTree(

                source_details.cvs_root, source_details.cvs_module,

                target_path)

        else:

            raise AssertionError(

                "unknown RCS type: %r" % source_details.rcstype)

    def archive(self, foreign_tree):

        """Archive the foreign tree."""

        local_name = 'foreign_tree.tar.gz'

        create_tarball(foreign_tree.local_path, 'foreign_tree.tar.gz')

        self.import_data_store.put(local_name)

    def fetch(self, target_path):

        """Fetch the foreign branch for `source_details` to `target_path`.

        If there is no tarball archived for `source_details`, then try to

        download (i.e. checkout) the foreign tree from its source repository,

        generally on a third party server.

        """

        try:

            return self.fetchFromArchive(target_path)

        except NoSuchFile:

            return self.fetchFromSource(target_path)

    def fetchFromSource(self, target_path):

        """Fetch the foreign tree for `source_details` to `target_path`."""

        branch = self._getForeignTree(target_path)

        branch.checkout()

        return branch

    def fetchFromArchive(self, target_path):

        """Fetch the foreign tree for `source_details` from the archive."""

        local_name = 'foreign_tree.tar.gz'

        if not self.import_data_store.fetch(local_name):

            raise NoSuchFile(local_name)

        extract_tarball(local_name, target_path)

        tree = self._getForeignTree(target_path)

        tree.update()

        return tree

class ImportWorker:

    """Oversees the actual work of a code import."""

    def __init__(self, source_details, logger, opener_policy):

        """Construct an `ImportWorker`.

        :param source_details: A `CodeImportSourceDetails` object.

        :param logger: A `Logger` to pass to cscvs.

        :param opener_policy: Policy object that decides what branches can

             be imported

        """

        self.source_details = source_details

        self._logger = logger

        self._opener_policy = opener_policy

    def getWorkingDirectory(self):

        """The directory we should change to and store all scratch files in.

        """

        base = config.codeimportworker.working_directory_root

        dirname = 'worker-for-branch-%s' % self.source_details.target_id

        return os.path.join(base, dirname)

    def run(self):

        """Run the code import job.

        This is the primary public interface to the `ImportWorker`. This

        method:

         1. Retrieves an up-to-date foreign tree to import.

         2. Gets the Bazaar branch to import into.

         3. Imports the foreign tree into the Bazaar branch. If we've

            already imported this before, we synchronize the imported Bazaar

            branch with the latest changes to the foreign tree.

         4. Publishes the newly-updated Bazaar branch, making it available to

            Launchpad users.

         5. Archives the foreign tree, so that we can update it quickly next

            time.

        """

        working_directory = self.getWorkingDirectory()

        if os.path.exists(working_directory):

            shutil.rmtree(working_directory)

        os.makedirs(working_directory)

        saved_pwd = os.getcwd()

        os.chdir(working_directory)

        try:

            return self._doImport()

        finally:

            shutil.rmtree(working_directory)

            os.chdir(saved_pwd)

    def _doImport(self):

        """Perform the import.

        :return: A CodeImportWorkerExitCode

        """

        raise NotImplementedError()

class ToBzrImportWorker(ImportWorker):

    """Oversees the actual work of a code import to Bazaar."""

    # Where the Bazaar working tree will be stored.

    BZR_BRANCH_PATH = 'bzr_branch'

    # Should `getBazaarBranch` create a working tree?

    needs_bzr_tree = True

    required_format = BzrDirFormat.get_default_format()

    def __init__(self, source_details, import_data_transport,

                 bazaar_branch_store, logger, opener_policy):

        """Construct a `ToBzrImportWorker`.

        :param source_details: A `CodeImportSourceDetails` object.

        :param bazaar_branch_store: A `BazaarBranchStore`. The import worker

            uses this to fetch and store the Bazaar branches that are created

            and updated during the import process.

        :param logger: A `Logger` to pass to cscvs.

        :param opener_policy: Policy object that decides what branches can

             be imported

        """

        super(ToBzrImportWorker, self).__init__(

            source_details, logger, opener_policy)

        self.bazaar_branch_store = bazaar_branch_store

        self.import_data_store = ImportDataStore(

            import_data_transport, self.source_details)

    def getBazaarBranch(self):

        """Return the Bazaar `Branch` that we are importing into."""

        if os.path.isdir(self.BZR_BRANCH_PATH):

            shutil.rmtree(self.BZR_BRANCH_PATH)

        return self.bazaar_branch_store.pull(

            self.source_details.target_id, self.BZR_BRANCH_PATH,

            self.required_format, self.needs_bzr_tree,

            stacked_on_url=self.source_details.stacked_on_url)

    def pushBazaarBranch(self, bazaar_branch):

        """Push the updated Bazaar branch to the server.

        :return: True if revisions were transferred.

        """

        return self.bazaar_branch_store.push(

            self.source_details.target_id, bazaar_branch,

            self.required_format,

            stacked_on_url=self.source_details.stacked_on_url)

class CSCVSImportWorker(ToBzrImportWorker):

    """An ImportWorker for imports that use CSCVS.

    As well as invoking cscvs to do the import, this class also needs to

    manage a foreign working tree.

    """

    # Where the foreign working tree will be stored.

    FOREIGN_WORKING_TREE_PATH = 'foreign_working_tree'