~vila/udd/717204-stop-too-fast

#!/usr/bin/python

import datetime
import errno
try:
    from hashlib import md5
except ImportError:
    from md5 import md5
import operator
import optparse
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import threading
import time

try:
    from debian import deb822, changelog
except ImportError:
    from debian_bundle import deb822, changelog

from launchpadlib.errors import HTTPError

from bzrlib import branch, bzrdir, debug, errors, revision as _mod_revision, tag, trace, transport, ui, urlutils
from bzrlib.plugin import load_plugins
from bzrlib.trace import mutter, log_exception_quietly, enable_default_logging

# We may not want to do this, but if there is a failure while loading plugins,
# it is the only way to find out. One option would be to call
# 'trace.pop_log_file()' immediately after loading plugins.
enable_default_logging()
mutter('import_package started w/ args: %r' % (sys.argv,))
load_plugins()

from bzrlib.plugins.builddeb import import_dsc

sys.path.insert(0, os.path.dirname(__file__))
import icommon

push_lock = threading.Lock()

parser = optparse.OptionParser()
parser.add_option("--push-only", dest="push", action="store_true")
parser.add_option("--no-push", dest="no_push", action="store_true")
parser.add_option("--check", dest="check", action="store_true")
parser.add_option("--no-existing", dest="no_existing", action="store_true")
parser.add_option("--extra-debian", dest="extra_debian")
parser.add_option("--verbose", dest="verbose", action="store_true")
parser.add_option("--keep-temp", dest="keep_temp", action="store_true",
                  help="Do not delete the temporary directory at "
                  "BASE_DIR/updates/PACKAGE_NAME when done")
parser.add_option("-D", dest="debug_flags", action="append", default=[],
                  help="Set debug flags, see 'bzr help debug-flags'")
parser.add_option("--lp-cache", dest="lp_cache", default=None,
                  help="Set the location of a Launchpadlib cache")
parser.add_option("--persistent-download-cache",
                  dest="persistent_download_cache", action="store_true",
                  help="Cache downloaded Debian source packages persistently "
                  "between runs of this script. Useful when testing and "
                  "developing to avoid repeatedly downloading the same files.")
parser.add_option("--local-branches", dest="local_branches", action="store_true",
                  help="Debugging mode in which branches in "
                  "BASE_DIR/localbranches/PACKAGE are used instead of the "
                  "real Launchpad branches. A local branch will be branched "
                  "from the Launchpad branch on first access if it does not "
                  "exist locally")

options, args = parser.parse_args()


class Killed(Exception):

    def __str__(self):
        return "Killed"


def die(signum, frame):
    push_lock.acquire()
    try:
        raise Killed
    finally:
        push_lock.release()


signal.signal(signal.SIGTERM, die)
signal.signal(signal.SIGINT, die)


def subprocess_setup():
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)


def update_lists():
    #for release in ("etch",):
    #    for component in ("main", "contrib", "non-free"):
    #        source_url = "http://ftp.uk.debian.org/debian/dists/%s/%s/source/Sources.gz" % (release, component)
    #        update_list(release, component, source_url)
    if icommon.lock_update_lists() is None:
        return
    for release in ("woody", "sarge", "etch",):
        for component in ("main", "contrib", "non-free"):
            source_url = "http://archive.debian.org/debian/dists/%s/%s/source/Sources.gz" % (release, component)
            icommon.update_list(release, component, source_url)


def pool_base(name):
    if name.startswith("lib"):
        return name[:4]
    return name[0]

distros = ("debian", "ubuntu",)

distro_list_base = {"ubuntu": "http://archive.ubuntu.com/ubuntu/dists/",
    "debian": "http://ftp.debian.org/dists/",
}

distro_components = {"ubuntu": ["main", "restricted", "universe", "multiverse"],
    "debian": ["main", "contrib", "non-free"],
}

distro_pool = {"ubuntu": "http://archive.ubuntu.com/ubuntu/",
    "debian": "http://ftp.debian.org/",
}

launchpad_base_url = "https://launchpad.net/"
ubuntu_base_url = launchpad_base_url + "ubuntu/"


lp = icommon.get_lp(options.lp_cache)
ubuntu = lp.distributions['ubuntu']
u_archive = ubuntu.main_archive
debian = lp.distributions['debian']
d_archive = debian.main_archive

lp_dist_o = {"ubuntu": ubuntu,
             "debian": debian}
lp_current_series_o = {}
lp_series_o = {}
def make_lp_current_series():
    for distro in distros:
        for series in lp_dist_o[distro].series:
            lp_series_o[series.name] = series
        lp_current_series_o[distro] = lp_dist_o[distro].current_series
make_lp_current_series()

lp_sp_o = {}
def get_lp_source_package(series, name):
    if name in lp_sp_o:
        if series in lp_sp_o[name]:
            return lp_sp_o[name][series]
    else:
        lp_sp_o[name] = {}
    lp_sp_o[name][series] = icommon.lp_call(series.getSourcePackage, name=name)
    return lp_sp_o[name][series]

missing_branches = {}


def get_debian_versions(package, extra_debian=None):
    vlist = icommon.ImportList()
    publications = icommon.lp_call(icommon.call_with_limited_size,
                d_archive.getPublishedSources,
                source_name=package, exact_match=True)
    pb = ui.ui_factory.nested_progress_bar()
    try:
        for idx, publication in enumerate(
                icommon.iterate_collection(publications)):
            pb.update('getting debian publications', idx, idx)
            assert publication.source_package_name == package
            release = publication.distro_series.name.lower()
            pocket = publication.pocket.lower()
            version = changelog.Version(publication.source_package_version)
            newp = icommon.PackageToImport(package, version, "debian", release,
                    pocket)
            vlist.add_if_needed(newp)
            # if idx >= 20:
            #     break
    finally:
        pb.finished()
    update_lists()
    try:
        proc = subprocess.Popen(["/usr/bin/madison-lite", "--mirror", icommon.lists_dir,
                package], stdout=subprocess.PIPE, preexec_fn=subprocess_setup)
    except OSError, e:
        if e.errno in (errno.ENOENT,):
            sys.stderr.write('You must have "madison-lite" installed.\n'
                             'try: sudo apt-get install madison-lite.\n')
            sys.exit(1)
        raise
    output, errout = proc.communicate()
    assert proc.returncode == 0, "Error running madison-lite"
    for line in output.splitlines():
        # package | version | release/component | arch
        ret_package = line[:line.index("|")].strip()
        assert ret_package == package
        line = line[line.index("|")+1:].strip()
        version = changelog.Version(line[:line.index("|")].strip())
        line = line[line.index("|")+1:].strip()
        release_component = line[:line.index("|")].strip()
        if "/" in release_component:
            release = release_component[:release_component.index("/")]
            component = release_component[release_component.index("/")+1:]
        else:
            release = release_component
            component = "main"
        newp = icommon.PackageToImport(package, version, "debian", release,
                "release", component=component, on_lp=False)
        vlist.add_if_needed(newp)
    if extra_debian is not None:
        f = open(extra_debian)
        try:
            for line in f:
                parts = line.split()
                version = changelog.Version(parts[0])
                release = parts[1]
                pocket =  parts[2]
                base_url =  parts[3]
                url = os.path.join(os.path.dirname(extra_debian), base_url)
                newp = icommon.PackageToImport(package, version, "debian",
                        release, pocket, url=url)
                vlist.add_if_needed(newp)
        finally:
            f.close()
    return vlist


def get_versions(package, extra_debian=None):
    vlist = get_debian_versions(package, extra_debian=extra_debian)
    publications = icommon.lp_call(icommon.call_with_limited_size,
                u_archive.getPublishedSources,
                source_name=package, exact_match=True)
    pb = ui.ui_factory.nested_progress_bar()
    try:
        for idx, publication in enumerate(
                icommon.iterate_collection(publications)):
            pb.update('getting ubuntu publications', idx, idx)
            assert publication.source_package_name == package
            release = publication.distro_series.name.lower()
            pocket = publication.pocket.lower()
            version = changelog.Version(publication.source_package_version)
            newp = icommon.PackageToImport(package, version, "ubuntu", release,
                    pocket)
            vlist.add_if_needed(newp)
            # if idx >= 20:
            #     break
    finally:
        pb.finished()
    vlist.sort()
    return vlist


def create_branch(path):
    to_transport = transport.get_transport(path)
    format = bzrdir.format_registry.make_bzrdir("2a")
    to_transport.ensure_base()
    new_branch = bzrdir.BzrDir.create_branch_convenience(
            to_transport.base, format=format,
            possible_transports=[to_transport])
    return new_branch

def create_updates_branch(updates_branch_path, importp, bstore, possible_transports=None):
    br_from = bstore.get_branch(importp, possible_transports=possible_transports)
    if br_from is None:
        updates_branch = create_branch(updates_branch_path)
        return updates_branch
    to_transport = transport.get_transport(updates_branch_path+"-origin")
    # Branch the branch in to the updates area as -origin for safe keeping
    br_from.lock_read()
    try:
        rev_id = br_from.last_revision()
        to_transport.mkdir('.')
        dir = br_from.bzrdir.sprout(to_transport.base,
            rev_id, possible_transports=[to_transport])
        origin_br = dir.open_branch()
        tag._merge_tags_if_possible(br_from, origin_br)
    finally:
        br_from.unlock()
    to_transport = transport.get_transport(updates_branch_path)
    # Branch that to the target path now
    origin_br.lock_read()
    try:
        to_transport.mkdir('.')
        dir = origin_br.bzrdir.sprout(to_transport.base,
            rev_id, possible_transports=[to_transport])
        updates_branch = dir.open_branch()
        tag._merge_tags_if_possible(origin_br, updates_branch)
    finally:
        origin_br.unlock()
    return updates_branch


def other_distro(distro):
    assert distro in ("debian", "ubuntu")
    if distro == "debian":
        return "ubuntu"
    else:
        return "debian"


def grab_file(base_url, name, target_dir, possible_transports=None):
    mutter("fetching %s/%s" % (base_url, name))
    local_path = os.path.join(target_dir, name)
    def get_file(transport):
        return transport.get(name)
    def redirected(t, e, redirection_notice):
        # _redirected_to has a bug that it doesn't support possible_transports,
        # so we just call get_transport directly, we know we are just getting a
        # file, so we don't have to try too hard.
        base, fname = urlutils.split(e.target)
        t2 = transport.get_transport(base,
                                     possible_transports=possible_transports)
        return t2
    # TODO: We could compute the md5sum while iterating the content, rather
    #       than re-reading the file.
    t = transport.get_transport(base_url,
                                possible_transports=possible_transports)
    location_f = transport.do_catching_redirections(get_file, t, redirected)
    try:
        local_f = open(local_path, "wb")
        try:
            shutil.copyfileobj(location_f, local_f)
        finally:
            local_f.close()
    finally:
        location_f.close()
    return local_path


def _check_md5(target_path, expected_md5sum, first):
    """See if the md5sum of the given file matches."""
    if not os.path.exists(target_path):
        return False
    file_md5 = md5()
    BUFSIZE = 128<<10
    f = open(target_path, 'rb')
    try:
        while True:
            b = f.read(BUFSIZE)
            if not b:
                break
            file_md5.update(b)
    finally:
        f.close()
    if file_md5.hexdigest() == expected_md5sum:
        if first:
            msg = 'reusing'
        else:
            msg = 'success'
        mutter('File at %s matched md5sum %s, %s.'
               % (target_path, expected_md5sum, msg))
        return True
    if first:
        msg = 'Redownloading'
    else:
        msg = 'Aborting'
    mutter('File at %s had md5sum %s expected %s. %s.'
           % (target_path, file_md5.hexdigest(), expected_md5sum, msg))
    return False


def dget(dsc_location, target_dir, possible_transports=None):
    base_url, dsc_name = urlutils.split(dsc_location)
    local_dsc_path = grab_file(base_url, dsc_name, target_dir,
                               possible_transports=possible_transports)
    dsc_f = open(local_dsc_path)
    try:
        dsc = deb822.Dsc(dsc_f)
        files = dsc['files']
        for file_info in files:
            name = file_info['name']
            target_path = os.path.join(target_dir, name)
            if _check_md5(target_path, file_info['md5sum'], first=True):
                # The file already exists, and the md5sum matches
                continue
            # We need to download the file
            grab_file(base_url, name, target_dir,
                      possible_transports=possible_transports)
            if not _check_md5(target_path, file_info['md5sum'], first=False):
                raise ValueError('The downloaded content for %s did'
                    ' not match the md5sum in %s'
                    % (target_path, local_dsc_path))
    finally:
        dsc_f.close()
    return local_dsc_path


def push_branch(br_from, to_loc, possible_transports=None, overwrite=False):
    created = False
    to_transport = transport.get_transport(to_loc,
                possible_transports=possible_transports)
    try:
        dir_to = bzrdir.BzrDir.open_from_transport(to_transport)
    except errors.NotBranchError:
        dir_to = None
    if dir_to is None:
        br_to = br_from.create_clone_on_transport(to_transport)
        created = True
    else:
        dir_to.push_branch(br_from, overwrite=overwrite)
    return created


def make_db_set(temp_dir, name, distro, release, pocket, updates_branch, bstore,
        possible_transports=None):
    db_set = import_dsc.DistributionBranchSet()
    for d in distros:
        for known_release in icommon.distro_releases[d]:
           for known_pocket in icommon.distro_pockets[d]:
               suite = icommon.make_suite(known_release, known_pocket)
               db = None
               if d == distro and known_release == release and known_pocket == pocket:
                   db = import_dsc.DistributionBranch(updates_branch, None)
                   db.tree = updates_branch.bzrdir.open_workingtree()
                   the_db = db
               else:
                   other_branch_path = os.path.join(temp_dir, suite)
                   try:
                       other_branch = branch.Branch.open(other_branch_path, possible_transports=possible_transports)
                   except errors.NotBranchError:
                       if (d, name, known_release, known_pocket) not in missing_branches:
                           other_branch = bstore.get_branch_parts(d, known_release, known_pocket, possible_transports=possible_transports)
                           if other_branch is None:
                               missing_branches[(d, name, known_release, known_pocket)] = True
                       else:
                           other_branch = None
                   if other_branch is not None:
                       db = import_dsc.DistributionBranch(other_branch, other_branch)
               if db is not None:
                   db_set.add_branch(db)
    return the_db


def extract_upstream_branch(update_db, upstream_dir):
    upstream_tip = None
    tags = update_db.branch.tags.get_tag_dict().items()
    timestamps = {}
    filtered_tags = []
    for tag, revid in tags:
        if not tag.startswith("upstream-"):
            continue
        try:
            revobj = update_db.branch.repository.get_revision(revid)
        except errors.NoSuchRevision:
            continue
        else:
            timestamp = revobj.timestamp
        timestamps[revid] = timestamp
        filtered_tags.append((tag, revid))
    filtered_tags.sort(key=lambda x: timestamps[x[1]])
    if len(filtered_tags) > 0:
        upstream_tip = filtered_tags[-1][1]
    if upstream_tip is not None:
        update_db._extract_upstream_tree(upstream_tip, upstream_dir)
    else:
        update_db._create_empty_upstream_tree(upstream_dir)


def mark_push_overwrite(temp_dir, importp):
    path = os.path.join(temp_dir,
            icommon.make_suite(importp.release, importp.pocket) + ".overwrite")
    f = open(path, 'wb')
    f.close()


def bug_description(distro, release, pocket, name, lp_side_branch_path):
    suite = icommon.make_suite(release, pocket)
    lp_path = "lp:" + lp_side_branch_path[lp_side_branch_path.index("~"):]
    description = ("The package history in the archive and the history "
            "in the bzr branch differ. As the archive is authoritative "
            "the history of lp:%s/%s/%s now reflects that and the old "
            "bzr branch has been pushed to %s. A merge should be performed "
            "if necessary." % (distro, suite, name, lp_path))
    return description


def file_bug(distro, release, pocket, name, lp_side_branch_path):
    suite = icommon.make_suite(release, pocket)
    description = bug_description(distro, release, pocket, name, lp_side_branch_path)
    udd = icommon.lp_call(operator.getitem, lp.projects, 'udd')
    new_bug = icommon.lp_call(lp.bugs.createBug, target=udd,
            title="Collision for %s %s" % (name, suite),
            description=description)
    mutter("Bug filed: %s" % str(new_bug.id))


def file_mp(distro, release, pocket, name, bstore, lp_side_branch_path):
    lp_url = icommon.SERVICE_ROOT \
              + lp_side_branch_path[lp_side_branch_path.index("~"):]
    lp_branch = icommon.lp_call(lp.load, lp_url)
    target_branch_url = bstore._branch_url(distro, release, pocket)
    target_lp_branch = icommon.lp_call(lp.load, target_branch_url)
    ubuntu_dev = icommon.lp_call(operator.getitem, lp.people, 'ubuntu-dev')
    mp = icommon.lp_call(lp_branch.createMergeProposal,
            target_branch=target_lp_branch,
            initial_comment=bug_description(distro, release, pocket, name,
                lp_side_branch_path),
            needs_review=False,
            #reviewers=[ubuntu_dev.self_link],
            )
    mutter("Merge proposal filed: %s" % str(mp.self_link))


def import_package(temp_dir, download_dir, importp, revid_db, bstore, possible_transports=None):
    suite = icommon.make_suite(importp.release, importp.pocket)
    if (importp.distro, importp.name, importp.release, importp.pocket) in missing_branches:
        del missing_branches[(importp.distro, importp.name, importp.release, importp.pocket)]
    updates_branch_path = os.path.join(temp_dir, suite)
    created_updates_branch = False
    try:
        updates_branch = branch.Branch.open(updates_branch_path, possible_transports=possible_transports)
    except errors.NotBranchError:
        updates_branch = create_updates_branch(updates_branch_path, importp,
                bstore, possible_transports=possible_transports)
        created_updates_branch = True
    last_version = revid_db.last_marked_version(suite)
    if last_version is not None:
        # Check that there aren't any revisions that would be lost
        tag_dict = updates_branch.tags.get_tag_dict()
        try:
            last_revid = tag_dict[last_version]
        except KeyError:
            assert False, "Doesn't seem to have version %s for %s %s at all" % (last_version, importp.version, suite)
        if last_revid != updates_branch.last_revision():
            # There's a real live collision! What do we do?
            # We first push the branch to LP for safe keeping
            lp_side_branch_path = bstore.side_branch_location(importp.distro, importp.release, suite)
            mutter("Collision for %s %s expecting last revision (%s) to be "
                    "%s, but it was %s instead, pushing to %s" % (importp.version, suite,
                        last_version, last_revid, updates_branch.last_revision(),
                        lp_side_branch_path))
            to_transport = transport.get_transport(lp_side_branch_path,
                    possible_transports=possible_transports)
            updates_branch.lock_read()
            try:
                to_transport.mkdir('.')
                dir = updates_branch.bzrdir.sprout(to_transport.base,
                    updates_branch.last_revision(),
                    possible_transports=possible_transports)
                lp_side_branch = dir.open_branch()
                tag._merge_tags_if_possible(updates_branch, lp_side_branch)
            finally:
                updates_branch.unlock()
            f = open(updates_branch_path + ".collisions", "a")
            try:
                f.write(lp_side_branch_path + "\n")
            finally:
                f.close()
            # Then we move it to one side and branch back the revision we want
            side_dir = tempfile.mkdtemp(dir=temp_dir)
            try:
                local_dir = os.path.join(side_dir, suite)
                os.rename(updates_branch_path, local_dir)
                side_branch = branch.Branch.open(local_dir)
                trunk_transport = transport.get_transport(updates_branch_path,
                        possible_transports=possible_transports)
                side_branch.lock_read()
                try:
                    trunk_transport.mkdir('.')
                    dir = side_branch.bzrdir.sprout(trunk_transport.base,
                        last_revid,
                        possible_transports=possible_transports)
                    updates_branch = dir.open_branch()
                    tag._merge_tags_if_possible(side_branch, updates_branch)
                finally:
                    side_branch.unlock()
            finally:
                shutil.rmtree(side_dir)
            mark_push_overwrite(temp_dir, importp)
    update_db = make_db_set(temp_dir, importp.name, importp.distro,
            importp.release, importp.pocket, updates_branch, bstore,
            possible_transports=possible_transports)
    # We remove any tag as it will only be there if there was a
    # collision, and then we need to remove the old tag to import
    # and set the tag for the new revision.
    # TODO: move the delete_tag in to a method on DistributionBranch.
    version_tag_name = update_db.tag_name(importp.version)
    if update_db.branch.tags.has_tag(version_tag_name):
        update_db.branch.tags.delete_tag(version_tag_name)
    if created_updates_branch:
        # Set the root id to be the same as another branch.
        other_branches = update_db.get_other_branches()
        if len(other_branches) > 0:
            other_branch = other_branches[-1]
            other_rev = other_branch.branch.last_revision()
            other_tree = other_branch.branch.repository.revision_tree(other_rev)
            root_id = other_tree.path2id("")
            update_db.tree.set_root_id(root_id)
    upstream_dir = tempfile.mkdtemp(dir=temp_dir)
    try:
        extract_upstream_branch(update_db, upstream_dir)
        dl_dir = tempfile.mkdtemp()
        try:
            local_dsc_path = dget(importp.get_url(), download_dir,
                                  possible_transports=possible_transports)
            update_db.import_package(local_dsc_path,
                    use_time_from_changelog=True)
        finally:
            shutil.rmtree(dl_dir)
    finally:
        shutil.rmtree(upstream_dir)
    mutter("Marking version %s in %s at revid %s" % (importp.version,
        suite, update_db.revid_of_version(importp.version)))
    revid_db.mark(importp.version,
            update_db.revid_of_version(importp.version), suite,
            update_db.branch)
    # Remove possible circular reference
    update_db.set_get_greater_branches_callback(None)
    update_db.set_get_lesser_branches_callback(None)
    del update_db


def find_unimported_versions(temp_dir, all_versions, name, revid_db, bstore,
        possible_transports=None):
    unimported_versions = icommon.ImportList()
    imported_suites = set()
    have_imported_old_debian = False
    last_target_info = None
    for importp in reversed(all_versions.plist):
        suite = icommon.make_suite(importp.release, importp.pocket)
        if (importp.distro, suite) in imported_suites:
            continue
        target_branch_path = os.path.join(temp_dir, suite)
        from_cache = False
        if last_target_info is not None:
            if target_branch_path == last_target_info[0]:
                target_branch = last_target_info[1]
                has_version = target_branch is not None
                from_cache = True
        if not from_cache:
            try:
                target_branch = branch.Branch.open(target_branch_path,
                        possible_transports=possible_transports)
                has_version = True
            except errors.NotBranchError:
                target_branch = bstore.get_branch(importp,
                        possible_transports=possible_transports)
                has_version = target_branch is not None
        last_target_info = (target_branch_path, target_branch)
        if has_version:
            db = import_dsc.DistributionBranch(target_branch, target_branch)
            has_version = db.has_version(importp.version)
        if has_version:
            revid = db.revid_of_version(importp.version)
            has_version = revid_db.check(importp.version, revid, suite, db.branch)
        if not has_version:
            # We'll never find old debian releases, so if there is a new one already
            # imported then don't try and re-import the old ones.
            if importp.distro == "debian" and importp.release not in icommon.lp_distro_releases[importp.distro]:
                if have_imported_old_debian:
                    imported_suites.add((importp.distro, suite))
                    continue
            assert not revid_db.is_marked(importp.version, suite), \
                "%s %s %s %s is marked but not imported" % (importp.name,
                        importp.version, importp.distro, importp.release)
            mutter("%s is not imported" % str(importp))
            unimported_versions.plist.insert(0, importp)
        else:
            mutter("%s %s is imported and so not checking for other "
                   "publications from there." % (importp.distro, suite))
            imported_suites.add((importp.distro, suite))
            if importp.distro == "debian":
                mutter("And as is is a Debian suite we will not look for "
                       "more in Debian")
                have_imported_old_debian = True
    return unimported_versions


def push_branches_back(package, temp_dir, bstore, possible_transports, local_branches):
    for d in distros:
        pushed_devel = False
        for known_release in icommon.lp_distro_releases[d]:
            for known_pocket in icommon.distro_pockets[d]:
                suite = icommon.make_suite(known_release, known_pocket)
                source_branch_path = os.path.join(temp_dir, suite)
                try:
                    source_branch = branch.Branch.open(source_branch_path)
                except errors.NotBranchError:
                    continue
                overwrite = False
                if os.path.exists(source_branch_path + ".overwrite"):
                    overwrite = True
                to_loc = bstore.push_location(d, known_release, known_pocket)
                if pushed_devel and not local_branches:
                    # XXX: Why? Is this a leftover from when Launchpad had a
                    # separate mirrored area? -- mbp 2011-02-08
                    mutter("sleeping to stack")
                    time.sleep(300)
                    pushed_devel = False
                push_lock.acquire()
                try:
                    created = push_branch(source_branch, to_loc,
                            possible_transports=possible_transports,
                            overwrite=overwrite)
                    try:
                        current = lp_current_series_o[d].name
                        lp_series = lp_series_o[known_release]
                        if (lp_series.name == current
                                and known_pocket == "release"
                                and created):
                            pushed_devel = True
                        bstore.set_official(d, known_release, known_pocket,
                                set_status=created,
                                set_reviewer=created and d == 'ubuntu')
                    except HTTPError, e:
                        print e.content.decode("utf-8", "replace")
                        raise
                    if os.path.exists(source_branch_path + ".collisions") and not local_branches:
                        mutter("sleeping to file merge proposal")
                        time.sleep(120)
                        f = open(source_branch_path + ".collisions")
                        try:
                            for line in f:
                                lp_side_branch_path = line.strip()
                                if d == "debian":
                                    file_bug(d, known_release, known_pocket, package,
                                            lp_side_branch_path)
                                else:
                                    file_mp(d, known_release, known_pocket, package,
                                            bstore, lp_side_branch_path)
                        finally:
                            f.close()
                finally:
                    push_lock.release()


def create_update_branches(package, versions, temp_dir, bstore, possible_transports=None):
    created_suites = set()
    for importp in versions.plist:
        suite = icommon.make_suite(importp.release, importp.pocket)
        if suite in created_suites:
            continue
        updates_branch_path = os.path.join(temp_dir, suite)
        try:
            updates_branch = branch.Branch.open(updates_branch_path, possible_transports=possible_transports)
        except errors.NotBranchError:
            updates_branch = create_updates_branch(updates_branch_path, importp,
                    bstore, possible_transports=possible_transports)
        created_suites.add(suite)


def _only_debian_dir_removals(changes):
    """Are there only directory deletions in this list of changes.

    For now, we also filter out changes outside of 'debian/'
    """
    for (file_id, paths, content_changed, versioneds, parent_ids,
         names, kinds, execs) in changes:
        if (kinds[0] != 'directory'
            or kinds[1] is not None
            or parent_ids[1] is not None
            or names[1] is not None
            or versioneds[1] is not False
            or not paths[0].startswith('debian/')):
            return False
    return True


def _are_trees_equivalent(pushed_tree, imported_tree):
    """Check if the contents of the two trees are identical."""
    changes = []
    for (file_id, paths, content_changed, versioneds, parent_ids,
         names, kinds, execs) in imported_tree.iter_changes(pushed_tree):
         if (not content_changed
             and versioneds[0] == versioneds[1]
             and parent_ids[0] == parent_ids[1]
             and names[0] == names[1]
             and kinds[0] == kinds[1]):
             # The only change for this object is the executable bit
             if paths[0].startswith('debian/patches/'):
                 # ignore this change
                 # when dpatch creates the file it sets executable=True (and at
                 # some other times)
                 # however the debian diff that we apply at import time, does
                 # not include information about executuable, so it gets
                 # accidentally lost sometimes.
                 # However, the trees *are* still considered identical, so
                 # don't include this change
                 mutter('ignoring changed exec bit from %s to %s for %s'
                        % (execs[0], execs[1], paths[0]))
                 continue
         changes.append((file_id, paths, content_changed, versioneds, parent_ids,
                         names, kinds, execs))
    if changes:
        # the diff.gz doesn't create empty directories, but people pushing
        # branches probably won't deleting them.
        # If the only changes present are deleting directories (no files, no
        # links, etc) then we know that it is only pruning empty dirs, and we
        # don't treat that as a change.
        if _only_debian_dir_removals(changes):
            trace.mutter('changes found, but only pruning empty directories:\n%s'
                         % ('\n'.join(map(str, changes)),))
            return True
        trace.mutter('Found changes between %s and %s:\n%s'
                     % (pushed_tree.get_revision_id(),
                        imported_tree.get_revision_id(),
                        '\n'.join(map(str, changes))))
        return False
    return True


def check_same(importp, db, revid, name, updates_branch, temp_dir, download_dir):
    mutter('Checking if content of revid: %s is the same as %s %s'
           % (revid, name, importp.version))
    lp_tree = db.branch.repository.revision_tree(revid)
    testdir = tempfile.mkdtemp(dir=temp_dir)
    try:
        test_branch = create_branch(os.path.join(testdir, name))
        test_up_branch = create_branch(os.path.join(testdir,
                    "%s-upstream" % name))
        test_db = import_dsc.DistributionBranch(test_branch,
            test_up_branch)
        test_db_set = import_dsc.DistributionBranchSet()
        test_db_set.add_branch(db)
        test_db_set.add_branch(test_db)
        test_db.tree = test_branch.bzrdir.open_workingtree()
        test_db.upstream_tree = test_up_branch.bzrdir.open_workingtree()
        local_dsc_path = dget(importp.get_url(), download_dir)
        test_db.import_package(local_dsc_path,
                use_time_from_changelog=True,
                file_ids_from=[lp_tree, updates_branch.basis_tree()],
                pull_debian=False)
        test_revid = test_db.revid_of_version(importp.version)
        test_tree = test_db.branch.repository.revision_tree(test_revid)
        return _are_trees_equivalent(lp_tree, test_tree)
    finally:
        shutil.rmtree(testdir)


def clean_collision(importp, suite, db, temp_dir, download_dir, name, updates_branch):
    mutter("Possible collision of %s %s %s" % (str(importp.version), importp.distro, suite))
    revid = db.revid_of_version(importp.version)
    db.branch.lock_read()
    try:
        return check_same(importp, db, revid, name, updates_branch, temp_dir, download_dir)
    finally:
        db.branch.unlock()


def clean_untagged_collision(importp, suite, db, temp_dir, download_dir, name,
        updates_branch, revid_db):
    # Check for the uploader not having tagged.
    # First we find the last revision that was marked for this series
    last_version = revid_db.last_marked_version(suite)
    tag_dict = updates_branch.tags.get_tag_dict()
    if last_version not in tag_dict:
        # Something odd is going on, but we'll deal with that later
        return False
    last_revid = tag_dict[last_version]
    if last_revid == updates_branch.last_revision():
        # There are no new revisions, so we have nothing to compare to
        return False
    # We want to compare against all mainline revisions since last_revid.
    # If last_revid is on the mainline then we compare everything since
    # If it isn't then we compare the one that merged it and all since
    stop_revid = icommon.find_earliest_merge(updates_branch, last_revid)
    if stop_revid is None:
        # Huh?
        return False
    if last_revid != stop_revid:
        # It wasn't on the mainline, so take an extra parent to check
        # this revision too
        parent_map = updates_branch.repository.get_parent_map([stop_revid])
        if stop_revid in parent_map:
            new_stop_revid = parent_map[stop_revid][0]
            if new_stop_revid != _mod_revision.NULL_REVISION:
                stop_revid = new_stop_revid
    tip_revid = updates_branch.last_revision()
    while tip_revid != stop_revid:
        # We compare tip_revid to what we would import
        if check_same(importp, db, tip_revid, name, updates_branch, temp_dir, download_dir):
            # We found where this import lives, tag it and tell the
            # caller
            db.tag_version(importp.version, revid=tip_revid)
            return True
        parent_map = updates_branch.repository.get_parent_map([tip_revid])
        if tip_revid not in parent_map:
            break
        tip_revid = parent_map[tip_revid][0]
        if tip_revid == _mod_revision.NULL_REVISION:
            break
    return False


def handle_collisions(name, versions, temp_dir, download_dir, revid_db,
        possible_transports=None):
    # We go through the reversed version list in case someone pushed a newer version
    filtered_versions = icommon.ImportList()
    filtered_suites = set()
    for importp in reversed(versions.plist):
        suite = icommon.make_suite(importp.release, importp.pocket)
        if suite in filtered_suites:
            continue
        updates_branch_path = os.path.join(temp_dir, suite)
        updates_branch = branch.Branch.open(updates_branch_path, possible_transports=possible_transports)
        db = import_dsc.DistributionBranch(updates_branch, updates_branch)
        db_set = import_dsc.DistributionBranchSet()
        db_set.add_branch(db)
        has_version = db.has_version(importp.version)
        if has_version:
            if clean_collision(importp, suite, db, temp_dir, download_dir, name, updates_branch):
                mutter("Clean collision")
                mutter("Marking version %s in %s at revid %s" % (importp.version,
                    suite, db.revid_of_version(importp.version)))
                revid_db.mark(importp.version,
                        db.revid_of_version(importp.version), suite,
                        db.branch)
                filtered_suites.add(suite)
                continue
        else:
            updates_branch.lock_write()
            try:
                if clean_untagged_collision(importp, suite, db, temp_dir, download_dir,
                        name, updates_branch, revid_db):
                    mutter("Clean untagged collision")
                    mutter("Marking version %s in %s at revid %s" % (importp.version,
                        suite, db.revid_of_version(importp.version)))
                    revid_db.mark(importp.version,
                            db.revid_of_version(importp.version), suite,
                            db.branch)
                    filtered_suites.add(suite)
                    continue
            finally:
                updates_branch.unlock()
        filtered_versions.plist.insert(0, importp)
    return filtered_versions


def main(package, push=True, check=False, extra_debian=None, no_existing=False,
        keep_temp=False, local_branches=False, persistent_download_cache=False):
    if not os.path.exists(icommon.logs_dir):
        os.makedirs(icommon.logs_dir)
    log_name = os.path.join(icommon.logs_dir, package)
    f = open(log_name, "ab", 1)
    log_token = trace.push_log_file(f)
    try:
        mutter("Time (UTC): %s" % str(datetime.datetime.utcnow()))
        f.flush()
        if not os.path.exists(icommon.updates_dir):
            os.makedirs(icommon.updates_dir)
        # The temp_dir is unique per single package import run.
        temp_dir = os.path.join(icommon.updates_dir, package)
        # The download_dir is used to dget Debian source packages. It can be
        # equal to the temp_dir to have these discarded when no longer
        # required, or set to a different directory to persistently store
        # downloaded source between runs.
        if persistent_download_cache:
            download_dir = os.path.join(icommon.download_cache_dir, package)
            if not os.path.exists(download_dir):
                os.makedirs(download_dir)
        else:
            download_dir = temp_dir
        if not no_existing:
            revid_db = icommon.RevidDatabase(icommon.sqlite_file, package)
            suffix = revid_db.get_suffix()
            if local_branches:
                bstore = icommon.LocalShadowBranchStore(package, lp, suffix)
            else:
                bstore = icommon.BranchStore(package, lp, suffix)
            commit_db = icommon.CommitDatabase(icommon.sqlite_file, package)
        else:
            revid_db = icommon.MemoryRevidDatabase(package)
            bstore = icommon.MemoryBranchStore(package, lp, "")
            commit_db = icommon.CommitDatabase(":memory:", package)
        def cleanup():
            if os.path.exists(temp_dir):
                shutil.rmtree(temp_dir)
        def push_back():
            possible_transports = []
            push_branches_back(package, temp_dir, bstore,
                    possible_transports, local_branches)
            icommon.generate_debian_diffs(lp, temp_dir, package, bstore,
                    possible_transports=possible_transports)
            icommon.generate_ubuntu_merges(lp, temp_dir, package, bstore,
                    possible_transports=possible_transports)
        if commit_db.has_commit_started():
            revid_db.cleanup_last_run(cleanup, push_back)
            commit_db.finish_commit()
        else:
            cleanup()
            revid_db.discard_last_run()
        os.mkdir(temp_dir)
        try:
            format = bzrdir.format_registry.make_bzrdir('2a')
            to_transport = transport.get_transport(temp_dir)
            to_transport.ensure_base()
            newdir = format.initialize_on_transport(to_transport)
            repo = newdir.create_repository(shared=True)
            repo.set_make_working_trees(True)
            mutter("finding all versions of %s" % (package,))
            versions = get_versions(package, extra_debian=extra_debian)
            assert len(versions.plist) > 0
            versions.reverse()
            mutter("found %d versions: %s" % (len(versions.plist), versions))
            possible_transports = []
            versions = find_unimported_versions(temp_dir, versions, package,
                    revid_db, bstore, possible_transports=possible_transports)
            mutter("These versions are new: %s" % str(versions))
            if len(versions.plist) == 0:
                icommon.possibly_generate_debian_diffs(lp, temp_dir, package,
                        bstore, possible_transports=possible_transports)
                icommon.possibly_generate_ubuntu_merges(lp, temp_dir, package,
                        bstore, possible_transports=possible_transports)
                print "Nothing new"
                if not keep_temp:
                    shutil.rmtree(temp_dir)
                return 0
            # First grab all the branches locally
            create_update_branches(package, versions, temp_dir, bstore,
                    possible_transports=possible_transports)
            # Then handle collisions
            versions = handle_collisions(package, versions, temp_dir,
                    download_dir, revid_db,
                    possible_transports=possible_transports)
            for importp in versions.plist:
                mutter("import %s to %s %s-%s (%s)" % (importp.get_url(),
                            importp.distro, importp.release, importp.pocket,
                            importp.component))
                import_package(temp_dir, download_dir, importp, revid_db, bstore, possible_transports=possible_transports)
            revid_db.commit_outstanding()
        except:
            if push and not keep_temp:
                shutil.rmtree(temp_dir)
            raise
        if check:
            from bzrlib.check import check_dwim
            check_dwim(temp_dir, False, do_repo=True)
        if push:
            commit_db.start_commit()
            push_branches_back(package, temp_dir, bstore, possible_transports, local_branches)
            icommon.generate_debian_diffs(lp, temp_dir, package, bstore,
                    possible_transports=possible_transports)
            icommon.generate_ubuntu_merges(lp, temp_dir, package, bstore,
                    possible_transports=possible_transports)
            revid_db.commit()
            if not keep_temp:
                shutil.rmtree(temp_dir)
        mutter("Imported %s" % str(versions))
        print "Imported %s" % str(versions)
        return 0
    except:
        log_exception_quietly()
        raise
    finally:
        trace.pop_log_file(log_token)
        f.close()

if __name__ == '__main__':
    if options.verbose:
        trace.set_verbosity_level(1)
        trace.push_log_file(sys.stderr)
    if sys.stderr.isatty():
        # Don't create a progress bar if we aren't actually in a terminal
        ui.ui_factory = ui.make_ui_for_terminal(sys.stdin, sys.stdout, sys.stderr)
    debug.debug_flags.update(options.debug_flags)


    if len(args) < 1:
        print "Must pass the name of a package"
        sys.exit(1)

    lock = icommon.lock_package(args[0])

    if lock is None:
        print "Unable to lock"
        sys.exit(icommon.no_lock_returncode)


    def push_back_main(args):
        if len(args) != 2:
            print "Must give name of package and dir"
            sys.exit(1)
        bstore = icommon.BranchStore(args[0], lp)
        try:
            sys.exit(push_branches_back(args[0], args[1], bstore, [], False))
        except HTTPError, e:
            print e.content.decode("utf-8", "replace")
            raise


    if options.push:
        push_back_main(args)
    else:
        if len(args) != 1:
            print "Must pass the name of one package to import"
            sys.exit(1)
        try:
            sys.exit(main(args[0], push=not options.no_push,
                        extra_debian=options.extra_debian, check=options.check,
                        no_existing=options.no_existing,
                        keep_temp=options.keep_temp,
                        local_branches=options.local_branches,
                        persistent_download_cache=options.persistent_download_cache))
        except HTTPError, e:
            print e.content.decode("utf-8", "replace")
            raise