~ubuntu-branches/ubuntu/raring/ubuntu-release-upgrader/raring

# DistUpgradeController.py 
#  
#  Copyright (c) 2004-2008 Canonical
#  
#  Author: Michael Vogt <michael.vogt@ubuntu.com>
# 
#  This program is free software; you can redistribute it and/or 
#  modify it under the terms of the GNU General Public License as 
#  published by the Free Software Foundation; either version 2 of the
#  License, or (at your option) any later version.
# 
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
# 
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
#  USA


from __future__ import absolute_import, print_function

import warnings
warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning)
import apt
import apt_pkg
import sys
import os
import subprocess
import locale
import logging
import shutil
import glob
import time
import copy
try:
    # >= 3.0
    from configparser import NoOptionError
    if sys.version >= '3.2':
        from configparser import ConfigParser as SafeConfigParser
    else:
        from configparser import SafeConfigParser
except ImportError:
    # < 3.0
    from ConfigParser import SafeConfigParser, NoOptionError
from .utils import (country_mirror,
                    url_downloadable,
                    check_and_fix_xbit,
                    get_arch,
                    iptables_active,
                    inside_chroot,
                    get_string_with_no_auth_from_source_entry,
                    is_child_of_process_name)
from string import Template
try:
    from urllib.parse import urlsplit
except ImportError:
    from urlparse import urlsplit

from .DistUpgradeView import STEP_PREPARE, STEP_MODIFY_SOURCES, STEP_FETCH, STEP_INSTALL, STEP_CLEANUP, STEP_REBOOT
from .DistUpgradeCache import MyCache
from .DistUpgradeConfigParser import DistUpgradeConfig
from .DistUpgradeQuirks import DistUpgradeQuirks
from .DistUpgradeAptCdrom import AptCdrom
from .DistUpgradeAufs import setupAufs, aufsOptionsAndEnvironmentSetup

# workaround broken relative import in python-apt (LP: #871007), we
# want the local version of distinfo.py from oneiric, but because of
# a bug in python-apt we will get the natty version that does not
# know about "Component.parent_component" leading to a crash
from . import distinfo
from . import sourceslist
sourceslist.DistInfo = distinfo.DistInfo

from .sourceslist import SourcesList, is_mirror
from .distro import get_distro, NoDistroTemplateException

from .DistUpgradeGettext import gettext as _
from .DistUpgradeGettext import ngettext
import gettext

from .DistUpgradeCache import (CacheExceptionDpkgInterrupted,
                               CacheExceptionLockingFailed,
                               NotEnoughFreeSpaceError)
from .DistUpgradeApport import run_apport

REBOOT_REQUIRED_FILE = "/var/run/reboot-required"


def component_ordering_key(a):
    """ key() function for sorted to ensure "correct" component ordering """
    ordering = ["main", "restricted", "universe", "multiverse"]
    try:
        return ordering.index(a)
    except ValueError:
        # ensure to sort behind the "official" components, order is not
        # really important for those
        return len(ordering)+1


class NoBackportsFoundException(Exception):
    pass


class DistUpgradeController(object):
    """ this is the controller that does most of the work """
    
    def __init__(self, distUpgradeView, options=None, datadir=None):
        # setup the paths
        localedir = "/usr/share/locale/"
        if datadir == None or datadir == '.':
            datadir = os.getcwd()
            localedir = os.path.join(datadir,"mo")
        self.datadir = datadir
        self.options = options

        # init gettext
        gettext.bindtextdomain("ubuntu-release-upgrader",localedir)
        gettext.textdomain("ubuntu-release-upgrader")

        # setup the view
        logging.debug("Using '%s' view" % distUpgradeView.__class__.__name__)
        self._view = distUpgradeView
        self._view.updateStatus(_("Reading cache"))
        self.cache = None

        if not self.options or self.options.withNetwork == None:
            self.useNetwork = True
        else:
            self.useNetwork = self.options.withNetwork
        if options:
            cdrompath = options.cdromPath
        else:
            cdrompath = None
        self.aptcdrom = AptCdrom(distUpgradeView, cdrompath)

        # the configuration
        self.config = DistUpgradeConfig(datadir)
        self.sources_backup_ext = "."+self.config.get("Files","BackupExt")

        # move some of the options stuff into the self.config, 
        # ConfigParser deals only with strings it seems *sigh*
        self.config.add_section("Options")
        self.config.set("Options","withNetwork", str(self.useNetwork))

        # aufs stuff
        aufsOptionsAndEnvironmentSetup(self.options, self.config)

        # some constants here
        self.fromDist = self.config.get("Sources","From")
        self.toDist = self.config.get("Sources","To")
        self.origin = self.config.get("Sources","ValidOrigin")
        self.arch = get_arch()

        # we run with --force-overwrite by default
        if "RELEASE_UPGRADE_NO_FORCE_OVERWRITE" not in os.environ:
            logging.debug("enable dpkg --force-overwrite")
            apt_pkg.config.set("DPkg::Options::","--force-overwrite")

        # we run in full upgrade mode by default
        self._partialUpgrade = False
        
        # install the quirks handler
        self.quirks = DistUpgradeQuirks(self, self.config)

        # setup env var 
        os.environ["RELEASE_UPGRADE_IN_PROGRESS"] = "1"
        os.environ["PYCENTRAL_FORCE_OVERWRITE"] = "1"
        os.environ["PATH"] = "%s:%s" % (os.getcwd()+"/imported",
                                        os.environ["PATH"])
        check_and_fix_xbit("./imported/invoke-rc.d")

        # set max retries
        maxRetries = self.config.getint("Network","MaxRetries")
        apt_pkg.config.set("Acquire::Retries", str(maxRetries))
        # max sizes for dpkgpm for large installs (see linux/limits.h and 
        #                                          linux/binfmts.h)
        apt_pkg.config.set("Dpkg::MaxArgs", str(64*1024))
        apt_pkg.config.set("Dpkg::MaxArgBytes", str(128*1024))

        # smaller to avoid hangs
        apt_pkg.config.set("Acquire::http::Timeout","20")
        apt_pkg.config.set("Acquire::ftp::Timeout","20")

        # no list cleanup here otherwise a "cancel" in the upgrade
        # will not restore the full state (lists will be missing)
        apt_pkg.config.set("Apt::Get::List-Cleanup", "false")

        # forced obsoletes
        self.forced_obsoletes = self.config.getlist("Distro","ForcedObsoletes")
        # list of valid mirrors that we can add
        self.valid_mirrors = self.config.getListFromFile("Sources","ValidMirrors")
        # third party mirrors
        self.valid_3p_mirrors = []
        if self.config.has_section('ThirdPartyMirrors'):
            self.valid_3p_mirrors = [pair[1] for pair in
                                     self.config.items('ThirdPartyMirrors')]
        # debugging
        #apt_pkg.config.set("DPkg::Options::","--debug=0077")

        # apt cron job
        self._aptCronJobPerms = 0o755

    def openCache(self, lock=True, restore_sources_list_on_fail=False):
        logging.debug("openCache()")
        if self.cache is None:
            self.quirks.run("PreCacheOpen")
        else:
            self.cache.release_lock()
            self.cache.unlock_lists_dir()
        # this loop will try getting the lock a couple of times
        MAX_LOCK_RETRIES = 20
        lock_retry = 0
        while True:
            try:
                # exit here once the cache is ready
                return self._openCache(lock)
            except CacheExceptionLockingFailed as e:
                # wait a bit
                lock_retry += 1
                self._view.processEvents()
                time.sleep(0.1)
                logging.debug(
                    "failed to lock the cache, retrying (%i)" % lock_retry)
                # and give up after some time
                if lock_retry > MAX_LOCK_RETRIES:
                    logging.error("Cache can not be locked (%s)" % e)
                    self._view.error(_("Unable to get exclusive lock"),
                                     _("This usually means that another "
                                       "package management application "
                                       "(like apt-get or aptitude) "
                                       "already running. Please close that "
                                       "application first."));
                    if restore_sources_list_on_fail:
                        self.abort()
                    else:
                        sys.exit(1)

    def _openCache(self, lock):
        try:
            self.cache = MyCache(self.config,
                                 self._view,
                                 self.quirks,
                                 self._view.getOpCacheProgress(),
                                 lock)
            # alias name for the plugin interface code
            self.apt_cache = self.cache
        # if we get a dpkg error that it was interrupted, just
        # run dpkg --configure -a
        except CacheExceptionDpkgInterrupted:
            logging.warning("dpkg interrupted, calling dpkg --configure -a")
            cmd = ["/usr/bin/dpkg","--configure","-a"]
            if os.environ.get("DEBIAN_FRONTEND") == "noninteractive":
                cmd.append("--force-confold")
            self._view.getTerminal().call(cmd)
            self.cache = MyCache(self.config,
                                 self._view,
                                 self.quirks,
                                 self._view.getOpCacheProgress())
        self.cache.partialUpgrade = self._partialUpgrade
        logging.debug("/openCache(), new cache size %i" % len(self.cache))

    def _viewSupportsSSH(self):
      """
      Returns True if this view support upgrades over ssh.
      In theory all views should support it, but for savety
      we do only allow text ssh upgrades (see LP: #322482)
      """
      supported = self.config.getlist("View","SupportSSH")
      if self._view.__class__.__name__ in supported:
          return True
      return False

    def _sshMagic(self):
        """ this will check for server mode and if we run over ssh.
            if this is the case, we will ask and spawn a additional
            daemon (to be sure we have a spare one around in case
            of trouble)
        """
        pidfile = os.path.join("/var/run/release-upgrader-sshd.pid")
        if (not os.path.exists(pidfile) and 
            os.path.isdir("/proc") and
            is_child_of_process_name("sshd")):
            # check if the frontend supports ssh upgrades (see lp: #322482)
            if not self._viewSupportsSSH():
                logging.error("upgrade over ssh not alllowed")
                self._view.error(_("Upgrading over remote connection not supported"),
                                 _("You are running the upgrade over a "
                                   "remote ssh connection with a frontend "
                                   "that does "
                                   "not support this. Please try a text "
                                   "mode upgrade with 'do-release-upgrade'."
                                   "\n\n"
                                   "The upgrade will "
                                   "abort now. Please try without ssh.")
                                 )
                sys.exit(1)
                return False
            # ask for a spare one to start (and below 1024)
            port = 1022
            res = self._view.askYesNoQuestion(
                _("Continue running under SSH?"),
                _("This session appears to be running under ssh. "
                  "It is not recommended to perform a upgrade "
                  "over ssh currently because in case of failure "
                  "it is harder to recover.\n\n"
                  "If you continue, an additional ssh daemon will be "
                  "started at port '%s'.\n"
                  "Do you want to continue?") % port)
            # abort
            if res == False:
                sys.exit(1)
            res = subprocess.call(["/usr/sbin/sshd",
                                   "-o", "PidFile=%s" % pidfile,
                                   "-p",str(port)])
            if res == 0:
                summary = _("Starting additional sshd")
                descr =  _("To make recovery in case of failure easier, an "
                           "additional sshd will be started on port '%s'. "
                           "If anything goes wrong with the running ssh "
                           "you can still connect to the additional one.\n"
                           ) % port
                if iptables_active():
                    cmd = "iptables -I INPUT -p tcp --dport %s -j ACCEPT" % port
                    descr += _(
                        "If you run a firewall, you may need to "
                        "temporarily open this port. As this is "
                        "potentially dangerous it's not done automatically. "
                        "You can open the port with e.g.:\n'%s'") % cmd
                self._view.information(summary, descr)
        return True

    def _tryUpdateSelf(self):
        """ this is a helper that is run if we are started from a CD
            and we have network - we will then try to fetch a update
            of ourself
        """  
        from .MetaRelease import MetaReleaseCore
        from .DistUpgradeFetcherSelf import DistUpgradeFetcherSelf
        # check if we run from a LTS 
        forceLTS=False
        if (self.release == "dapper" or
            self.release == "hardy" or
            self.release == "lucid" or
            self.release == "precise"):
            forceLTS=True
        m = MetaReleaseCore(useDevelopmentRelease=False,
                            forceLTS=forceLTS)
        # this will timeout eventually
        while m.downloading:
            self._view.processEvents()
            time.sleep(0.1)
        if m.new_dist is None:
            logging.error("No new dist found")
            return False
        # we have a new dist
        progress = self._view.getAcquireProgress()
        fetcher = DistUpgradeFetcherSelf(new_dist=m.new_dist,
                                         progress=progress,
                                         options=self.options,
                                         view=self._view)
        fetcher.run()

    def _pythonSymlinkCheck(self):
        """ sanity check that /usr/bin/python points to the default
            python version. Users tend to modify this symlink, which
            breaks stuff in obscure ways (Ubuntu #75557).
        """
        logging.debug("_pythonSymlinkCheck run")
        if os.path.exists('/usr/share/python/debian_defaults'):
            config = SafeConfigParser()
            config.readfp(open('/usr/share/python/debian_defaults'))
            try:
                expected_default = config.get('DEFAULT', 'default-version')
            except NoOptionError:
                logging.debug("no default version for python found in '%s'" % config)
                return False
            try:
                fs_default_version = os.readlink('/usr/bin/python')
            except OSError as e:
                logging.error("os.readlink failed (%s)" % e)
                return False
            if not fs_default_version in (expected_default, os.path.join('/usr/bin', expected_default)):
                logging.debug("python symlink points to: '%s', but expected is '%s' or '%s'" % (fs_default_version, expected_default, os.path.join('/usr/bin', expected_default)))
                return False
        return True


    def prepare(self):
        """ initial cache opening, sanity checking, network checking """
        # first check if that is a good upgrade
        self.release = release = subprocess.Popen(["lsb_release","-c","-s"],
                                   stdout=subprocess.PIPE,
                                   universal_newlines=True).communicate()[0].strip()
        logging.debug("lsb-release: '%s'" % release)
        if not (release == self.fromDist or release == self.toDist):
            logging.error("Bad upgrade: '%s' != '%s' " % (release, self.fromDist))
            self._view.error(_("Can not upgrade"),
                             _("An upgrade from '%s' to '%s' is not "
                               "supported with this tool." % (release, self.toDist)))
            sys.exit(1)

        # setup aufs
        if self.config.getWithDefault("Aufs", "EnableFullOverlay", False):
            aufs_rw_dir = self.config.get("Aufs","RWDir")
            if not setupAufs(aufs_rw_dir):
                logging.error("aufs setup failed")
                self._view.error(_("Sandbox setup failed"),
                                 _("It was not possible to create the sandbox "
                                   "environment."))
                return False

            # all good, tell the user about the sandbox mode
            logging.info("running in aufs overlay mode")
            self._view.information(_("Sandbox mode"),
                                   _("This upgrade is running in sandbox "
                                     "(test) mode. All changes are written "
                                     "to '%s' and will be lost on the next "
                                     "reboot.\n\n"
                                     "*No* changes written to a system directory "
                                     "from now until the next reboot are "
                                     "permanent.") % aufs_rw_dir)

        # setup backports (if we have them)
        if self.options and self.options.havePrerequists:
            backportsdir = os.getcwd()+"/backports"
            logging.info("using backports in '%s' " % backportsdir)
            logging.debug("have: %s" % glob.glob(backportsdir+"/*.udeb"))
            if os.path.exists(backportsdir+"/usr/bin/dpkg"):
                apt_pkg.config.set("Dir::Bin::dpkg",backportsdir+"/usr/bin/dpkg");
            if os.path.exists(backportsdir+"/usr/lib/apt/methods"):
                apt_pkg.config.set("Dir::Bin::methods",backportsdir+"/usr/lib/apt/methods")
            conf = backportsdir+"/etc/apt/apt.conf.d/01ubuntu"
            if os.path.exists(conf):
                logging.debug("adding config '%s'" % conf)
                apt_pkg.read_config_file(apt_pkg.config, conf)

        # do the ssh check and warn if we run under ssh
        self._sshMagic()
        # check python version
        if not self._pythonSymlinkCheck():
            logging.error("pythonSymlinkCheck() failed, aborting")
            self._view.error(_("Can not upgrade"),
                             _("Your python install is corrupted. "
                               "Please fix the '/usr/bin/python' symlink."))
            sys.exit(1)
        # open cache
        try:
            self.openCache()
        except SystemError as e:
            logging.error("openCache() failed: '%s'" % e)
            return False
        if not self.cache.sanity_check(self._view):
            return False

        # now figure out if we need to go into desktop or 
        # server mode - we use a heuristic for this
        self.serverMode = self.cache.need_server_mode()
        if self.serverMode:
            os.environ["RELEASE_UPGRADE_MODE"] = "server"
        else:
            os.environ["RELEASE_UPGRADE_MODE"] = "desktop"

        if not self.checkViewDepends():
            logging.error("checkViewDepends() failed")
            return False

        if os.path.exists("/usr/bin/debsig-verify"):
            logging.error("debsig-verify is installed")
            self._view.error(_("Package 'debsig-verify' is installed"),
                             _("The upgrade can not continue with that "
                               "package installed.\n"
                               "Please remove it with synaptic "
                               "or 'apt-get remove debsig-verify' first "
                               "and run the upgrade again."))
            self.abort()

        from .DistUpgradeMain import SYSTEM_DIRS
        for systemdir in SYSTEM_DIRS:
            if os.path.exists(systemdir) and not os.access(systemdir, os.W_OK):
                logging.error("%s not writable" % systemdir)
                self._view.error(
                    _("Can not write to '%s'") % systemdir,
                    _("Its not possible to write to the system directory "
                      "'%s' on your system. The upgrade can not "
                      "continue.\n"
                      "Please make sure that the system directory is "
                      "writable.") % systemdir)
                self.abort()
            

        # FIXME: we may try to find out a bit more about the network
        # connection here and ask more  intelligent questions
        if self.aptcdrom and self.options and self.options.withNetwork == None:
            res = self._view.askYesNoQuestion(_("Include latest updates from the Internet?"),
                                              _("The upgrade system can use the internet to "
                                                "automatically download "
                                                "the latest updates and install them during the "
                                                "upgrade.  If you have a network connection this is "
                                                "highly recommended.\n\n"
                                                "The upgrade will take longer, but when "
                                                "it is complete, your system will be fully up to "
                                                "date.  You can choose not to do this, but you "
                                                "should install the latest updates soon after "
                                                "upgrading.\n"
                                                "If you answer 'no' here, the network is not "
                                                "used at all."),
                                              'Yes')
            self.useNetwork = res
            self.config.set("Options","withNetwork", str(self.useNetwork))
            logging.debug("useNetwork: '%s' (selected by user)" % res)
            if res:
                self._tryUpdateSelf()
        return True

    def _sourcesListEntryDownloadable(self, entry):
        """
        helper that checks if a sources.list entry points to 
        something downloadable
        """
        logging.debug("verifySourcesListEntry: %s" % entry)
        # no way to verify without network
        if not self.useNetwork:
            logging.debug("skiping downloadable check (no network)")
            return True
        # check if the entry points to something we can download
        uri = "%s/dists/%s/Release" % (entry.uri, entry.dist)
        return url_downloadable(uri, logging.debug)

    def rewriteSourcesList(self, mirror_check=True):
        logging.debug("rewriteSourcesList()")

        sync_components = self.config.getlist("Sources","Components")

        # skip mirror check if special environment is set
        # (useful for server admins with internal repos)
        if (self.config.getWithDefault("Sources","AllowThirdParty",False) or
            "RELEASE_UPRADER_ALLOW_THIRD_PARTY" in os.environ):
            logging.warning("mirror check skipped, *overriden* via config")
            mirror_check=False

        # check if we need to enable main
        if mirror_check == True and self.useNetwork:
            # now check if the base-meta pkgs are available in
            # the archive or only available as "now"
            # -> if not that means that "main" is missing and we
            #    need to  enable it
            for pkgname in self.config.getlist("Distro","BaseMetaPkgs"):
                if ((not pkgname in self.cache or
                     not self.cache[pkgname].candidate or
                     len(self.cache[pkgname].candidate.origins) == 0)
                    or
                    (self.cache[pkgname].candidate and
                     len(self.cache[pkgname].candidate.origins) == 1 and
                     self.cache[pkgname].candidate.origins[0].archive == "now")
                   ):
                    logging.debug("BaseMetaPkg '%s' has no candidate.origins" % pkgname)
                    try:
                        distro = get_distro()
                        distro.get_sources(self.sources)
                        distro.enable_component("main")
                    except NoDistroTemplateException:
                        # fallback if everything else does not work,
                        # we replace the sources.list with a single
                        # line to ubuntu-main
                        logging.warning('get_distro().enable_component("man") failed, overwriting sources.list instead as last resort')
                        s =  "# auto generated by ubuntu-release-upgrader"
                        s += "deb http://archive.ubuntu.com/ubuntu %s main restricted" % self.toDist
                        s += "deb http://archive.ubuntu.com/ubuntu %s-updates main restricted" % self.toDist
                        s += "deb http://security.ubuntu.com/ubuntu %s-security main restricted" % self.toDist
                        open("/etc/apt/sources.list","w").write(s)
                    break
            
        # this must map, i.e. second in "from" must be the second in "to"
        # (but they can be different, so in theory we could exchange
        #  component names here)
        pockets = self.config.getlist("Sources","Pockets")
        fromDists = [self.fromDist] + ["%s-%s" % (self.fromDist, x) 
                                       for x in pockets]
        toDists = [self.toDist] + ["%s-%s" % (self.toDist,x) 
                                   for x in pockets]
        self.sources_disabled = False

        # look over the stuff we have
        foundToDist = False
        # collect information on what components (main,universe) are enabled for what distro (sub)version
        # e.g. found_components = { 'hardy':set("main","restricted"), 'hardy-updates':set("main") }
        self.found_components = {}
        for entry in self.sources.list[:]:

            # ignore invalid records or disabled ones
            if entry.invalid or entry.disabled:
                continue
            
            # we disable breezy cdrom sources to make sure that demoted
            # packages are removed
            if entry.uri.startswith("cdrom:") and entry.dist == self.fromDist:
                logging.debug("disabled '%s' cdrom entry (dist == fromDist)" % entry)
                entry.disabled = True
                continue
            # check if there is actually a lists file for them available
            # and disable them if not
            elif entry.uri.startswith("cdrom:"):
                # 
                listdir = apt_pkg.config.find_dir("Dir::State::lists")
                if not os.path.exists("%s/%s%s_%s_%s" % 
                                      (listdir,
                                       apt_pkg.uri_to_filename(entry.uri),
                                       "dists",
                                       entry.dist,
                                       "Release")):
                    logging.warning("disabling cdrom source '%s' because it has no  Release file" % entry)
                    entry.disabled = True
                continue

            # special case for archive.canonical.com that needs to
            # be rewritten (for pre-gutsy upgrades)
            cdist = "%s-commercial" % self.fromDist
            if (not entry.disabled and
                entry.uri.startswith("http://archive.canonical.com") and
                entry.dist == cdist):
                entry.dist = self.toDist
                entry.comps = ["partner"]
                logging.debug("transitioned commercial to '%s' " % entry)
                continue

            # special case for landscape.canonical.com because they
            # don't use a standard archive layout (gutsy->hardy)
            if (not entry.disabled and
                entry.uri.startswith("http://landscape.canonical.com/packages/%s" % self.fromDist)):
                logging.debug("commenting landscape.canonical.com out")
                entry.disabled = True
                continue
            
            # handle upgrades from a EOL release and check if there
            # is a supported release available
            if (not entry.disabled and
                "old-releases.ubuntu.com/" in entry.uri):
                logging.debug("upgrade from old-releases.ubuntu.com detected")
                # test country mirror first, then archive.u.c
                for uri in ["http://%sarchive.ubuntu.com/ubuntu" % country_mirror(),
                            "http://archive.ubuntu.com/ubuntu"]:
                    test_entry = copy.copy(entry)
                    test_entry.uri = uri
                    test_entry.dist = self.toDist
                    if self._sourcesListEntryDownloadable(test_entry):
                        logging.info("transition from old-release.u.c to %s" % uri)
                        entry.uri = uri
                        break

            logging.debug("examining: '%s'" %  get_string_with_no_auth_from_source_entry(entry))
            # check if it's a mirror (or official site)
            validMirror = self.isMirror(entry.uri)
            thirdPartyMirror = not mirror_check or self.isThirdPartyMirror(entry.uri)
            if validMirror or thirdPartyMirror:
                # disabled/security/commercial/extras are special cases
                # we use validTo/foundToDist to figure out if we have a 
                # main archive mirror in the sources.list or if we 
                # need to add one
                validTo = True
                if (entry.disabled or
                    entry.type == "deb-src" or
                    "/security.ubuntu.com" in entry.uri or
                    "%s-security" % self.fromDist in entry.dist or
                    "%s-backports" % self.fromDist in entry.dist or
                    "/archive.canonical.com" in entry.uri or
                    "/extras.ubuntu.com" in entry.uri):
                    validTo = False
                if entry.dist in toDists:
                    # so the self.sources.list is already set to the new
                    # distro
                    logging.debug("entry '%s' is already set to new dist" % get_string_with_no_auth_from_source_entry(entry))
                    foundToDist |= validTo
                elif entry.dist in fromDists:
                    foundToDist |= validTo
                    entry.dist = toDists[fromDists.index(entry.dist)]
                    logging.debug("entry '%s' updated to new dist" % get_string_with_no_auth_from_source_entry(entry))
                elif entry.type == 'deb-src':
                    continue
                elif validMirror:
                    # disable all entries that are official but don't
                    # point to either "to" or "from" dist
                    entry.disabled = True
                    self.sources_disabled = True
                    logging.debug("entry '%s' was disabled (unknown dist)" % get_string_with_no_auth_from_source_entry(entry))

                # if we make it to this point, we have a official or third-party mirror

                # check if the arch is powerpc or sparc and if so, transition
                # to ports.ubuntu.com (powerpc got demoted in gutsy, sparc
                # in hardy)
                if (entry.type == "deb" and
                    not "ports.ubuntu.com" in entry.uri and
                    (self.arch == "powerpc" or self.arch == "sparc")):
                    logging.debug("moving %s source entry to 'ports.ubuntu.com' " % self.arch)
                    entry.uri = "http://ports.ubuntu.com/ubuntu-ports/"

                # gather what components are enabled and are inconsitent
                for d in ["%s" % self.toDist,
                          "%s-updates" % self.toDist,
                          "%s-security" % self.toDist]:
                    # create entry if needed, ignore disabled
                    # entries and deb-src
                    self.found_components.setdefault(d,set())
                    if (not entry.disabled and entry.dist == d and
                        entry.type == "deb"):
                        for comp in entry.comps:
                            # only sync components we know about
                            if not comp in sync_components:
                                continue
                            self.found_components[d].add(comp)

            else:
                # disable anything that is not from a official mirror or a whitelisted third party
                if entry.dist == self.fromDist:
                    entry.dist = self.toDist
                disable_comment = " " + _("disabled on upgrade to %s") % self.toDist
                if isinstance(entry.comment, bytes):
                    entry.comment += disable_comment.encode('UTF-8')
                else:
                    entry.comment += disable_comment
                entry.disabled = True
                self.sources_disabled = True
                logging.debug("entry '%s' was disabled (unknown mirror)" % get_string_with_no_auth_from_source_entry(entry))

        # now go over the list again and check for missing components 
        # in $dist-updates and $dist-security and add them
        for entry in self.sources.list[:]:
            # skip all comps that are not relevant (including e.g. "hardy")
            if (entry.invalid or entry.disabled or entry.type == "deb-src" or 
                entry.uri.startswith("cdrom:") or entry.dist == self.toDist):
                continue
            # now check for "$dist-updates" and "$dist-security" and add any inconsistencies
            if entry.dist in self.found_components:
                component_diff = self.found_components[self.toDist]-self.found_components[entry.dist]
                if component_diff:
                    logging.info("fixing components inconsistency from '%s'" % get_string_with_no_auth_from_source_entry(entry))
                    # extend and make sure to keep order
                    entry.comps.extend(
                        sorted(component_diff, key=component_ordering_key))
                    logging.info("to new entry '%s'" % get_string_with_no_auth_from_source_entry(entry))
                    del self.found_components[entry.dist]
        return foundToDist

    def updateSourcesList(self):
        logging.debug("updateSourcesList()")
        self.sources = SourcesList(matcherPath=".")
        if not self.rewriteSourcesList(mirror_check=True):
            logging.error("No valid mirror found")
            res = self._view.askYesNoQuestion(_("No valid mirror found"),
                             _("While scanning your repository "
                               "information no mirror entry for "
                               "the upgrade was found. "
                               "This can happen if you run a internal "
                               "mirror or if the mirror information is "
                               "out of date.\n\n"
                               "Do you want to rewrite your "
                               "'sources.list' file anyway? If you choose "
                               "'Yes' here it will update all '%s' to '%s' "
                               "entries.\n"
                               "If you select 'No' the upgrade will cancel."
                               ) % (self.fromDist, self.toDist))
            if res:
                # re-init the sources and try again
                self.sources = SourcesList(matcherPath=".")
                # its ok if rewriteSourcesList fails here if
                # we do not use a network, the sources.list may be empty
                if (not self.rewriteSourcesList(mirror_check=False)
                    and self.useNetwork):
                    #hm, still nothing useful ...
                    prim = _("Generate default sources?")
                    secon = _("After scanning your 'sources.list' no "
                              "valid entry for '%s' was found.\n\n"
                              "Should default entries for '%s' be "
                              "added? If you select 'No', the upgrade "
                              "will cancel.") % (self.fromDist, self.toDist)
                    if not self._view.askYesNoQuestion(prim, secon):
                        self.abort()

                    # add some defaults here
                    # FIXME: find mirror here
                    logging.info("generate new default sources.list")
                    uri = "http://archive.ubuntu.com/ubuntu"
                    comps = ["main","restricted"]
                    self.sources.add("deb", uri, self.toDist, comps)
                    self.sources.add("deb", uri, self.toDist+"-updates", comps)
                    self.sources.add("deb",
                                     "http://security.ubuntu.com/ubuntu/",
                                     self.toDist+"-security", comps)
            else:
                self.abort()

        # write (well, backup first ;) !
        self.sources.backup(self.sources_backup_ext)
        self.sources.save()

        # re-check if the written self.sources are valid, if not revert and
        # bail out
        # TODO: check if some main packages are still available or if we
        #       accidentally shot them, if not, maybe offer to write a standard
        #       sources.list?
        try:
            sourceslist = apt_pkg.SourceList()
            sourceslist.read_main_list()
        except SystemError:
            logging.error("Repository information invalid after updating (we broke it!)")
            self._view.error(_("Repository information invalid"),
                             _("Upgrading the repository information "
                               "resulted in a invalid file so a bug "
                               "reporting process is being started."))
            subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
            return False

        if self.sources_disabled:
            self._view.information(_("Third party sources disabled"),
                             _("Some third party entries in your sources.list "
                               "were disabled. You can re-enable them "
                               "after the upgrade with the "
                               "'software-properties' tool or "
                               "your package manager."
                               ))
        return True

    def _logChanges(self):
        # debugging output
        logging.debug("About to apply the following changes")
        inst = []
        up = []
        rm = []
        held = []
        keep = []
        for pkg in self.cache:
            if pkg.marked_install: inst.append(pkg.name)
            elif pkg.marked_upgrade: up.append(pkg.name)
            elif pkg.marked_delete: rm.append(pkg.name)
            elif (pkg.is_installed and pkg.is_upgradable): held.append(pkg.name)
            elif pkg.is_installed and pkg.marked_keep: keep.append(pkg.name)
        logging.debug("Keep at same version: %s" % " ".join(keep))
        logging.debug("Upgradable, but held- back: %s" % " ".join(held))
        logging.debug("Remove: %s" % " ".join(rm))
        logging.debug("Install: %s" % " ".join(inst))
        logging.debug("Upgrade: %s" % " ".join(up))
        

    def doPostInitialUpdate(self):
        # check if we have packages in ReqReinst state that are not
        # downloadable
        logging.debug("doPostInitialUpdate")
        self.quirks.run("PostInitialUpdate")
        if not self.cache:
            return False
        if len(self.cache.req_reinstall_pkgs) > 0:
            logging.warning("packages in reqReinstall state, trying to fix")
            self.cache.fix_req_reinst(self._view)
            self.openCache()
        if len(self.cache.req_reinstall_pkgs) > 0:
            reqreinst = self.cache.req_reinstall_pkgs
            header = ngettext("Package in inconsistent state",
                              "Packages in inconsistent state",
                              len(reqreinst))
            summary = ngettext("The package '%s' is in an inconsistent "
                               "state and needs to be reinstalled, but "
                               "no archive can be found for it. "
                               "Please reinstall the package manually "
                               "or remove it from the system.",
                               "The packages '%s' are in an inconsistent "
                               "state and need to be reinstalled, but "
                               "no archive can be found for them. "
                               "Please reinstall the packages manually "
                               "or remove them from the system.",
                               len(reqreinst)) % ", ".join(reqreinst)
            self._view.error(header, summary)
            return False
        # FIXME: check out what packages are downloadable etc to
        # compare the list after the update again
        self.obsolete_pkgs = self.cache._getObsoletesPkgs()
        self.foreign_pkgs = self.cache._getForeignPkgs(self.origin, self.fromDist, self.toDist)
        if self.serverMode:
            self.tasks = self.cache.installedTasks
        logging.debug("Foreign: %s" % " ".join(self.foreign_pkgs))
        logging.debug("Obsolete: %s" % " ".join(self.obsolete_pkgs))
        return True

    def doUpdate(self, showErrors=True, forceRetries=None):
        logging.debug("running doUpdate() (showErrors=%s)" % showErrors)
        if not self.useNetwork:
            logging.debug("doUpdate() will not use the network because self.useNetwork==false")
            return True
        self.cache._list.read_main_list()
        progress = self._view.getAcquireProgress()
        # FIXME: also remove all files from the lists partial dir!
        currentRetry = 0
        if forceRetries is not None:
            maxRetries=forceRetries
        else:
            maxRetries = self.config.getint("Network","MaxRetries")
        while currentRetry < maxRetries:
            try:
                self.cache.update(progress)
            except (SystemError, IOError) as e:
                logging.error("IOError/SystemError in cache.update(): '%s'. Retrying (currentRetry: %s)" % (e,currentRetry))
                currentRetry += 1
                continue
            # no exception, so all was fine, we are done
            return True

        logging.error("doUpdate() failed completely")
        if showErrors:
            self._view.error(_("Error during update"),
                             _("A problem occurred during the update. "
                               "This is usually some sort of network "
                               "problem, please check your network "
                               "connection and retry."), "%s" % e)
        return False


    def _checkFreeSpace(self):
        " this checks if we have enough free space on /var and /usr"
        err_sum = _("Not enough free disk space")
        err_long= _("The upgrade has aborted. "
                    "The upgrade needs a total of %s free space on disk '%s'. "
                    "Please free at least an additional %s of disk "
                    "space on '%s'. "
                    "Empty your trash and remove temporary "
                    "packages of former installations using "
                    "'sudo apt-get clean'.")
        # allow override
        if self.config.getWithDefault("FreeSpace","SkipCheck",False):
            logging.warning("free space check skipped via config override")
            return True
        # do the check
        with_snapshots = self._is_apt_btrfs_snapshot_supported()
        try:
            self.cache.checkFreeSpace(with_snapshots)
        except NotEnoughFreeSpaceError as e:
            # ok, showing multiple error dialog sucks from the UI
            # perspective, but it means we do not need to break the
            # string freeze
            for required in e.free_space_required_list:
                self._view.error(err_sum, err_long % (required.size_total,
                                                      required.dir,
                                                      required.size_needed,
                                                      required.dir))
            return False
        return True


    def askDistUpgrade(self):
        self._view.updateStatus(_("Calculating the changes"))
        if not self.cache.distUpgrade(self._view, self.serverMode, self._partialUpgrade):
            return False

        if self.serverMode:
            if not self.cache.installTasks(self.tasks):
                return False

        # show changes and confirm
        changes = self.cache.get_changes()
        self._view.processEvents()

        # log the changes for debugging
        self._logChanges()
        self._view.processEvents()

        # check if we have enough free space 
        if not self._checkFreeSpace():
            return False
        self._view.processEvents()

        # get the demotions
        self.installed_demotions = self.cache.get_installed_demoted_packages()
        if len(self.installed_demotions) > 0:
            self.installed_demotions.sort()
            logging.debug("demoted: '%s'" % " ".join([x.name for x in self.installed_demotions]))
            logging.debug("found components: %s" % self.found_components)

        # flush UI
        self._view.processEvents()

        # ask the user
        res = self._view.confirmChanges(_("Do you want to start the upgrade?"),
                                        changes,
                                        self.installed_demotions,
                                        self.cache.required_download)
        return res

    def _disableAptCronJob(self):
        if os.path.exists("/etc/cron.daily/apt"):
            #self._aptCronJobPerms = os.stat("/etc/cron.daily/apt")[ST_MODE]
            logging.debug("disabling apt cron job (%s)" % oct(self._aptCronJobPerms))
            os.chmod("/etc/cron.daily/apt",0o644)
    def _enableAptCronJob(self):
        if os.path.exists("/etc/cron.daily/apt"):
            logging.debug("enabling apt cron job")
            os.chmod("/etc/cron.daily/apt", self._aptCronJobPerms)

    def doDistUpgradeFetching(self):
        # ensure that no apt cleanup is run during the download/install
        self._disableAptCronJob()
        # get the upgrade
        currentRetry = 0
        fprogress = self._view.getAcquireProgress()
        #iprogress = self._view.getInstallProgress(self.cache)
        # start slideshow
        url = self.config.getWithDefault("Distro","SlideshowUrl",None)
        if url:
            try:
                lang = locale.getdefaultlocale()[0].split('_')[0]
            except:
                logging.exception("getdefaultlocale")
                lang = "en"
            self._view.getHtmlView().open("%s#locale=%s" % (url, lang))
        # retry the fetching in case of errors
        maxRetries = self.config.getint("Network","MaxRetries")
        # FIXME: we get errors like 
        #   "I wasn't able to locate file for the %s package" 
        #  here sometimes. its unclear why and not reproducible, the 
        #  current theory is that for some reason the file is not
        #  considered trusted at the moment 
        #  pkgAcquireArchive::QueueNext() runs debReleaseIndex::IsTrused()
        #  (the later just checks for the existence of the .gpg file)
        #  OR 
        #  the fact that we get a pm and fetcher here confuses something
        #  in libapt?
        # POSSIBLE workaround: keep the list-dir locked so that 
        #          no apt-get update can run outside from the release
        #          upgrader 
        user_canceled = False
        # LP: #1102593 - In Python 3, the targets of except clauses get `del`d
        # from the current namespace after the exception is handled, so we
        # must assign it to a different variable in order to use it after
        # the while loop.
        exception = None
        while currentRetry < maxRetries:
            try:
                pm = apt_pkg.PackageManager(self.cache._depcache)
                fetcher = apt_pkg.Acquire(fprogress)
                self.cache._fetch_archives(fetcher, pm)
            except apt.cache.FetchCancelledException as e:
                logging.info("user canceled")
                user_canceled = True
                exception = e
                break
            except IOError as e:
                # fetch failed, will be retried
                logging.error("IOError in cache.commit(): '%s'. Retrying (currentTry: %s)" % (e,currentRetry))
                currentRetry += 1
                exception = e
                continue
            return True

        # maximum fetch-retries reached without a successful commit
        if user_canceled:
            self._view.information(_("Upgrade canceled"),
                                   _("The upgrade will cancel now and the "
                                     "original system state will be restored. "
                                     "You can resume the upgrade at a later "
                                     "time."))
        else:
            logging.error("giving up on fetching after maximum retries")
            self._view.error(_("Could not download the upgrades"),
                             _("The upgrade has aborted. Please check your "
                               "Internet connection or "
                               "installation media and try again. All files "
                               "downloaded so far have been kept."),
                             "%s" % exception)
        # abort here because we want our sources.list back
        self._enableAptCronJob()
        self.abort()

    def _is_apt_btrfs_snapshot_supported(self):
        """ check if apt-btrfs-snapshot is usable """
        try:
            import apt_btrfs_snapshot
        except ImportError:
            return
        try:
            apt_btrfs = apt_btrfs_snapshot.AptBtrfsSnapshot()
            res = apt_btrfs.snapshots_supported()
        except:
            logging.exception("failed to check btrfs support")
            return False
        logging.debug("apt btrfs snapshots supported: %s" % res)
        return res

    def _maybe_create_apt_btrfs_snapshot(self):
        """ create btrfs snapshot (if btrfs layout is there) """
        if not self._is_apt_btrfs_snapshot_supported():
            return
        import apt_btrfs_snapshot
        apt_btrfs = apt_btrfs_snapshot.AptBtrfsSnapshot()
        prefix = "release-upgrade-%s-" % self.toDist
        res = apt_btrfs.create_btrfs_root_snapshot(prefix)
        logging.info("creating snapshot '%s' (success=%s)" % (prefix, res))

    def doDistUpgrade(self):
        # add debug code only here
        #apt_pkg.config.set("Debug::pkgDpkgPM", "1")
        #apt_pkg.config.set("Debug::pkgOrderList", "1")
        #apt_pkg.config.set("Debug::pkgPackageManager", "1")

        # get the upgrade
        currentRetry = 0
        fprogress = self._view.getAcquireProgress()
        iprogress = self._view.getInstallProgress(self.cache)
        # retry the fetching in case of errors
        maxRetries = self.config.getint("Network","MaxRetries")
        if not self._partialUpgrade:
            self.quirks.run("StartUpgrade")
            # FIXME: take this into account for diskspace calculation
            self._maybe_create_apt_btrfs_snapshot()
        res = False
        while currentRetry < maxRetries:
            try:
                res = self.cache.commit(fprogress,iprogress)
                logging.debug("cache.commit() returned %s" % res)
            except SystemError as e:
                logging.error("SystemError from cache.commit(): %s" % e)
                # if its a ordering bug we can cleanly revert to
                # the previous release, no packages have been installed
                # yet (LP: #328655, #356781)
                if os.path.exists("/var/run/ubuntu-release-upgrader-apt-exception"):
                    e = open("/var/run/ubuntu-release-upgrader-apt-exception").read()
                    logging.error("found exception: '%s'" % e)
                    # if its a ordering bug we can cleanly revert but we need to write
                    # a marker for the parent process to know its this kind of error
                    pre_configure_errors = [
                        "E:Internal Error, Could not perform immediate configuration",
                        "E:Couldn't configure pre-depend "]
                    for preconf_error in pre_configure_errors:
                        if str(e).startswith(preconf_error):
                            logging.debug("detected preconfigure error, restorting state")
                            self._enableAptCronJob()
                            # FIXME: strings are not good, but we are in string freeze
                            # currently
                            msg = _("Error during commit")
                            msg += "\n'%s'\n" % str(e)
                            msg += _("Restoring original system state")
                            self._view.error(_("Could not install the upgrades"), msg)
                            # abort() exits cleanly
                            self.abort()
                
                # invoke the frontend now and show a error message
                msg = _("The upgrade has aborted. Your system "
                        "could be in an unusable state. A recovery "
                        "will run now (dpkg --configure -a).")
                if not self._partialUpgrade:
                    if not run_apport():
                        msg += _("\n\nPlease report this bug in a browser at "
                                 "http://bugs.launchpad.net/ubuntu/+source/ubuntu-release-upgrader/+filebug "
                                 "and attach the files in /var/log/dist-upgrade/ "
                                 "to the bug report.\n"
                                 "%s" % e)
                self._view.error(_("Could not install the upgrades"), msg)
                # installing the packages failed, can't be retried
                cmd = ["/usr/bin/dpkg","--configure","-a"]
                if os.environ.get("DEBIAN_FRONTEND") == "noninteractive":
                    cmd.append("--force-confold")
                self._view.getTerminal().call(cmd)
                self._enableAptCronJob()
                return False
            except IOError as e:
                # fetch failed, will be retried
                logging.error("IOError in cache.commit(): '%s'. Retrying (currentTry: %s)" % (e,currentRetry))
                currentRetry += 1
                continue
            except OSError as e:
                logging.exception("cache.commit()")
                # deal gracefully with:
                #  OSError: [Errno 12] Cannot allocate memory
                if e.errno == 12:
                    self._enableAptCronJob()
                    msg = _("Error during commit")
                    msg += "\n'%s'\n" % str(e)
                    msg += _("Restoring original system state")
                    self._view.error(_("Could not install the upgrades"), msg)
                    # abort() exits cleanly
                    self.abort()
            # no exception, so all was fine, we are done
            self._enableAptCronJob()
            return True
        
        # maximum fetch-retries reached without a successful commit
        logging.error("giving up on fetching after maximum retries")
        self._view.error(_("Could not download the upgrades"),
                         _("The upgrade has aborted. Please check your "\
                           "Internet connection or "\
                           "installation media and try again. "),
                           "%s" % e)
        # abort here because we want our sources.list back
        self.abort()

    def doPostUpgrade(self):
        # reopen cache
        self.openCache()
        # run the quirks handler that does does like things adding
        # missing groups or similar work arounds, only do it on real
        # upgrades
        self.quirks.run("PostUpgrade")
        # check out what packages are cruft now
        # use self.{foreign,obsolete}_pkgs here and see what changed
        now_obsolete = self.cache._getObsoletesPkgs()
        now_foreign = self.cache._getForeignPkgs(self.origin, self.fromDist, self.toDist)
        logging.debug("Obsolete: %s" % " ".join(now_obsolete))
        logging.debug("Foreign: %s" % " ".join(now_foreign))
        # now sanity check - if a base meta package is in the obsolete list now, that means
        # that something went wrong (see #335154) badly with the network. this should never happen, but it did happen
        # at least once so we add extra paranoia here
        for pkg in self.config.getlist("Distro","BaseMetaPkgs"):
            if pkg in now_obsolete:
                logging.error("the BaseMetaPkg '%s' is in the obsolete list, something is wrong, ignoring the obsoletes" % pkg)
                now_obsolete = set()
                break
        # check if we actually want obsolete removal
        if not self.config.getWithDefault("Distro","RemoveObsoletes", True):
            logging.debug("Skipping obsolete Removal")
            return True

        # now get the meta-pkg specific obsoletes and purges
        for pkg in self.config.getlist("Distro","MetaPkgs"):
            if pkg in self.cache and self.cache[pkg].is_installed:
                self.forced_obsoletes.extend(self.config.getlist(pkg,"ForcedObsoletes"))
        # now add the obsolete kernels to the forced obsoletes
        self.forced_obsoletes.extend(self.cache.identifyObsoleteKernels())
        logging.debug("forced_obsoletes: %s", self.forced_obsoletes)

        # mark packages that are now obsolete (and where not obsolete
        # before) to be deleted. make sure to not delete any foreign
        # (that is, not from ubuntu) packages
        if self.useNetwork:
            # we can only do the obsoletes calculation here if we use a
            # network. otherwise after rewriting the sources.list everything
            # that is not on the CD becomes obsolete (not-downloadable)
            remove_candidates = now_obsolete - self.obsolete_pkgs
        else:
            # initial remove candidates when no network is used should
            # be the demotions to make sure we don't leave potential
            # unsupported software
            remove_candidates = set([p.name for p in self.installed_demotions])
        remove_candidates |= set(self.forced_obsoletes)

        # no go for the unused dependencies
        unused_dependencies = self.cache._getUnusedDependencies()
        logging.debug("Unused dependencies: %s" %" ".join(unused_dependencies))
        remove_candidates |= set(unused_dependencies)

        # see if we actually have to do anything here
        if not self.config.getWithDefault("Distro","RemoveObsoletes", True):
            logging.debug("Skipping RemoveObsoletes as stated in the config")
            remove_candidates = set()
        logging.debug("remove_candidates: '%s'" % remove_candidates)
        logging.debug("Start checking for obsolete pkgs")
        progress = self._view.getOpCacheProgress()
        for (i, pkgname) in enumerate(remove_candidates):
            progress.update((i/float(len(remove_candidates)))*100.0)
            if pkgname not in self.foreign_pkgs:
                self._view.processEvents()
                if not self.cache.tryMarkObsoleteForRemoval(pkgname, remove_candidates, self.foreign_pkgs):
                    logging.debug("'%s' scheduled for remove but not safe to remove, skipping", pkgname)
        logging.debug("Finish checking for obsolete pkgs")
        progress.done()

        # get changes
        changes = self.cache.get_changes()
        logging.debug("The following packages are marked for removal: %s" % " ".join([pkg.name for pkg in changes]))
        summary = _("Remove obsolete packages?")
        actions = [_("_Keep"), _("_Remove")]
        # FIXME Add an explanation about what obsolete packages are
        #explanation = _("")
        if (len(changes) > 0 and 
            self._view.confirmChanges(summary, changes, [], 0, actions, False)):
            fprogress = self._view.getAcquireProgress()
            iprogress = self._view.getInstallProgress(self.cache)
            try:
                self.cache.commit(fprogress,iprogress)
            except (SystemError, IOError) as e:
                logging.error("cache.commit() in doPostUpgrade() failed: %s" % e)
                self._view.error(_("Error during commit"),
                                 _("A problem occurred during the clean-up. "
                                   "Please see the below message for more "
                                   "information. "),
                                   "%s" % e)
        # run stuff after cleanup
        self.quirks.run("PostCleanup")
        # run the post upgrade scripts that can do fixup like xorg.conf
        # fixes etc - only do on real upgrades
        if not self._partialUpgrade:
            self.runPostInstallScripts()
        return True

    def runPostInstallScripts(self):
        """ 
        scripts that are run in any case after the distupgrade finished 
        whether or not it was successful
        """
        # now run the post-upgrade fixup scripts (if any)
        for script in self.config.getlist("Distro","PostInstallScripts"):
            if not os.path.exists(script):
                logging.warning("PostInstallScript: '%s' not found" % script)
                continue
            logging.debug("Running PostInstallScript: '%s'" % script)
            try:
                # work around kde tmpfile problem where it eats permissions
                check_and_fix_xbit(script)
                self._view.getTerminal().call([script], hidden=True)
            except Exception as e:
                logging.error("got error from PostInstallScript %s (%s)" % (script, e))
        
    def abort(self):
        """ abort the upgrade, cleanup (as much as possible) """
        logging.debug("abort called")
        if hasattr(self, "sources"):
            self.sources.restore_backup(self.sources_backup_ext)
        if hasattr(self, "aptcdrom"):
            self.aptcdrom.restore_backup(self.sources_backup_ext)
        # generate a new cache
        self._view.updateStatus(_("Restoring original system state"))
        self._view.abort()
        self.openCache()
        sys.exit(1)

    def _checkDep(self, depstr):
        " check if a given depends can be satisfied "
        for or_group in apt_pkg.parse_depends(depstr):
            logging.debug("checking: '%s' " % or_group)
            for dep in or_group:
                depname = dep[0]
                ver = dep[1]
                oper = dep[2]
                if depname not in self.cache:
                    logging.error("_checkDep: '%s' not in cache" % depname)
                    return False
                inst = self.cache[depname]
                instver = getattr(inst.installed, "version", None)
                if (instver != None and
                    apt_pkg.check_dep(instver,oper,ver) == True):
                    return True
        logging.error("depends '%s' is not satisfied" % depstr)
        return False
                
    def checkViewDepends(self):
        " check if depends are satisfied "
        logging.debug("checkViewDepends()")
        res = True
        # now check if anything from $foo-updates is required
        depends = self.config.getlist("View","Depends")
        depends.extend(self.config.getlist(self._view.__class__.__name__,
                                           "Depends"))
        for dep in depends:
            logging.debug("depends: '%s'", dep)
            res &= self._checkDep(dep)
            if not res:
                # FIXME: instead of error out, fetch and install it
                #        here
                self._view.error(_("Required depends is not installed"),
                                 _("The required dependency '%s' is not "
                                   "installed. " % dep))
                sys.exit(1)
        return res 

    def _verifyBackports(self):
        # run update (but ignore errors in case the countrymirror
        # substitution goes wrong, real errors will be caught later
        # when the cache is searched for the backport packages)
        backportslist = self.config.getlist("PreRequists","Packages")
        i=0
        noCache = apt_pkg.config.find("Acquire::http::No-Cache","false")
        maxRetries = self.config.getint("Network","MaxRetries")
        while i < maxRetries:
            self.doUpdate(showErrors=False)
            self.openCache()
            for pkgname in backportslist:
                if pkgname not in self.cache:
                    logging.error("Can not find backport '%s'" % pkgname)
                    raise NoBackportsFoundException(pkgname)
            if self._allBackportsAuthenticated(backportslist):
                break
            # FIXME: move this to some more generic place
            logging.debug("setting a cache control header to turn off caching temporarily")
            apt_pkg.config.set("Acquire::http::No-Cache","true")
            i += 1
        if i == maxRetries:
            logging.error("pre-requists item is NOT trusted, giving up")
            return False
        apt_pkg.config.set("Acquire::http::No-Cache",noCache)
        return True

    def _allBackportsAuthenticated(self, backportslist):
        # check if the user overwrote the check
        if apt_pkg.config.find_b("APT::Get::AllowUnauthenticated",False) == True:
            logging.warning("skip authentication check because of APT::Get::AllowUnauthenticated==true")
            return True
        try:
            b = self.config.getboolean("Distro","AllowUnauthenticated")
            if b:
                return True
        except NoOptionError:
            pass
        for pkgname in backportslist:
            pkg = self.cache[pkgname]
            if not pkg.candidate:
                return False
            for cand in pkg.candidate.origins:
                if cand.trusted:
                    break
            else:
                return False
        return True

    def isMirror(self, uri):
        """ check if uri is a known mirror """
        # deal with username:password in a netloc
        raw_uri = uri.rstrip("/")
        scheme, netloc, path, query, fragment = urlsplit(raw_uri)
        if "@" in netloc:
            netloc = netloc.split("@")[1]
        # construct new mirror url without the username/pw
        uri = "%s://%s%s" % (scheme, netloc, path)
        for mirror in self.valid_mirrors:
            mirror = mirror.rstrip("/")
            if is_mirror(mirror, uri):
                return True
            # deal with mirrors like
            #    deb http://localhost:9977/security.ubuntu.com/ubuntu intrepid-security main restricted
            # both apt-debtorrent and apt-cacher use this (LP: #365537)
            mirror_host_part = mirror.split("//")[1]
            if uri.endswith(mirror_host_part):
                logging.debug("found apt-cacher/apt-torrent style uri %s" % uri)
                return True
        return False

    def isThirdPartyMirror(self, uri):
        " check if uri is a whitelisted third-party mirror "
        uri = uri.rstrip("/")
        for mirror in self.valid_3p_mirrors:
            mirror = mirror.rstrip("/")
            if is_mirror(mirror, uri):
                return True
        return False

    def _getPreReqMirrorLines(self, dumb=False):
        " get sources.list snippet lines for the current mirror "
        lines = ""
        sources = SourcesList(matcherPath=".")
        for entry in sources.list:
            if entry.invalid or entry.disabled:
                continue
            if (entry.type == "deb" and 
                entry.disabled == False and
                self.isMirror(entry.uri) and
                "main" in entry.comps and
                "%s-updates" % self.fromDist in entry.dist and
                not entry.uri.startswith("http://security.ubuntu.com") and
                not entry.uri.startswith("http://archive.ubuntu.com") ):
                new_line = "deb %s %s-updates main\n" % (entry.uri, self.fromDist)
                if not new_line in lines:
                    lines += new_line
            # FIXME: do we really need "dumb" mode?
            #if (dumb and entry.type == "deb" and
            #    "main" in entry.comps):
            #    lines += "deb %s %s-proposed main\n" % (entry.uri, self.fromDist)
        return lines

    def _addPreRequistsSourcesList(self, template, out, dumb=False):
        " add prerequists based on template into the path outfile "
        # go over the sources.list and try to find a valid mirror
        # that we can use to add the backports dir
        logging.debug("writing prerequists sources.list at: '%s' " % out)
        outfile = open(out, "w")
        mirrorlines = self._getPreReqMirrorLines(dumb)
        for line in open(template):
            template = Template(line)
            outline = template.safe_substitute(mirror=mirrorlines)
            outfile.write(outline)
            logging.debug("adding '%s' prerequists" % outline)
        outfile.close()
        return True

    def getRequiredBackports(self):
        " download the backports specified in DistUpgrade.cfg "
        logging.debug("getRequiredBackports()")
        res = True
        backportsdir = os.path.join(os.getcwd(),"backports")
        if not os.path.exists(backportsdir):
            os.mkdir(backportsdir)
        backportslist = self.config.getlist("PreRequists","Packages")

        # FIXME: this needs to be ported
        # if we have them on the CD we are fine
        if self.aptcdrom and not self.useNetwork:
            logging.debug("Searching for pre-requists on CDROM")
            p = os.path.join(self.aptcdrom.cdrompath,
                             "dists/stable/main/dist-upgrader/binary-%s/" % apt_pkg.config.find("APT::Architecture"))
            found_pkgs = set()
            for deb in glob.glob(p+"*_*.deb"):
                logging.debug("found pre-req '%s' to '%s'" % (deb, backportsdir))
                found_pkgs.add(os.path.basename(deb).split("_")[0])
            # now check if we got all backports on the CD
            if not set(backportslist).issubset(found_pkgs):
                logging.error("Expected backports: '%s' but got '%s'" % (set(backportslist), found_pkgs))
                return False
            # now install them
            self.cache.release_lock()
            p = subprocess.Popen(
                ["/usr/bin/dpkg", "-i", ] + glob.glob(p+"*_*.deb"),
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                universal_newlines=True)
            res = None
            while res is None:
                res = p.poll()
                self._view.pulseProgress()
                time.sleep(0.02)
            self._view.pulseProgress(finished=True)
            self.cache.get_lock()
            logging.info("installing backport debs exit code '%s'" % res)
            logging.debug("dpkg output:\n%s" % p.communicate()[0])
            if res != 0:
                return False
            # and re-start itself when it done
            return self.setupRequiredBackports()

        # we support PreRequists/SourcesList-$arch sections here too
        # 
        # logic for mirror finding works list this:     
        # - use the mirror template from the config, then: [done]
        # 
        #  - try to find known mirror (isMirror) and prepend it [done]
        #  - archive.ubuntu.com is always a fallback at the end [done]
        # 
        # see if we find backports with that
        # - if not, try guessing based on URI, Trust and Dist   [done]
        #   in existing sources.list (internal mirror with no
        #   outside connection maybe)
        # 
        # make sure to remove file on cancel
        
        # FIXME: use the DistUpgradeFetcherCore logic
        #        in mirror_from_sources_list() here
        #        (and factor that code out into a helper)

        conf_option = "SourcesList"
        if self.config.has_option("PreRequists",conf_option+"-%s" % self.arch):
            conf_option = conf_option + "-%s" % self.arch
        prereq_template = self.config.get("PreRequists",conf_option)
        if not os.path.exists(prereq_template):
            logging.error("sourceslist not found '%s'" % prereq_template)
            return False
        outpath = os.path.join(apt_pkg.config.find_dir("Dir::Etc::sourceparts"), prereq_template)
        outfile = os.path.join(apt_pkg.config.find_dir("Dir::Etc::sourceparts"), prereq_template)
        self._addPreRequistsSourcesList(prereq_template, outfile) 
        try:
            self._verifyBackports()
        except NoBackportsFoundException as e:
            self._addPreRequistsSourcesList(prereq_template, outfile, dumb=True) 
            try:
                self._verifyBackports()
            except NoBackportsFoundException as e:
                logging.warning("no backport for '%s' found" % e)
            return False
        
        # FIXME: sanity check the origin (just for safety)
        for pkgname in backportslist:
            pkg = self.cache[pkgname]
            # look for the right version (backport)
            ver = self.cache._depcache.get_candidate_ver(pkg._pkg)
            if not ver:
                logging.error("No candidate for '%s'" % pkgname)
                os.unlink(outpath)
                return False
            if ver.file_list == None:
                logging.error("No ver.file_list for '%s'" % pkgname)
                os.unlink(outpath)
                return False
            logging.debug("marking '%s' for install" % pkgname)
            # mark install
            pkg.mark_install(auto_inst=False, auto_fix=False)

        # now get it
        res = False
        try:
            res = self.cache.commit(self._view.getAcquireProgress(),
                                    self._view.getInstallProgress(self.cache))
        except IOError as e:
            logging.error("fetch_archives returned '%s'" % e)
            res = False
        except SystemError as e:
            logging.error("install_archives returned '%s'" % e)
            res = False

        if res == False:
            logging.warning("_fetch_archives for backports returned False")

        # all backports done, remove the pre-requirests.list file again
        try:
            os.unlink(outfile)
        except Exception as e:
            logging.error("failed to unlink pre-requists file: '%s'" % e)
        return self.setupRequiredBackports()

    # used by both cdrom/http fetcher
    def setupRequiredBackports(self):
        # ensure that the new release upgrader uses the latest python-apt
        # from the backport path
        os.environ["PYTHONPATH"] = "/usr/lib/release-upgrader-python-apt"
        # copy log so that it gets not overwritten
        logging.shutdown()
        shutil.copy("/var/log/dist-upgrade/main.log",
                    "/var/log/dist-upgrade/main_pre_req.log")
        # now exec self again
        args = sys.argv + ["--have-prerequists"]
        if self.useNetwork:
            args.append("--with-network")
        else:
            args.append("--without-network")
        logging.info("restarting upgrader")
        #print("restarting upgrader to make use of the backports")
        # work around kde being clever and removing the x bit
        check_and_fix_xbit(sys.argv[0])
        os.execve(sys.argv[0],args, os.environ)

    # this is the core
    def fullUpgrade(self):
        # sanity check (check for ubuntu-desktop, brokenCache etc)
        self._view.updateStatus(_("Checking package manager"))
        self._view.setStep(STEP_PREPARE)

        if not self.prepare():
            logging.error("self.prepared() failed")
            self._view.error(_("Preparing the upgrade failed"),
                             _("Preparing the system for the upgrade "
                               "failed so a bug reporting process is "
                               "being started."))
            subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
            sys.exit(1)

        # mvo: commented out for now, see #54234, this needs to be
        #      refactored to use a arch=any tarball
        if (self.config.has_section("PreRequists") and
            self.options and
            self.options.havePrerequists == False):
            logging.debug("need backports")
            # get backported packages (if needed)
            if not self.getRequiredBackports():
                self._view.error(_("Getting upgrade prerequisites failed"),
                                 _("The system was unable to get the "
                                   "prerequisites for the upgrade. "
                                   "The upgrade will abort now and restore "
                                   "the original system state.\n"
                                   "\n"
                                   "Additionally, a bug reporting process is "
                                   "being started."))
                subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
                self.abort()

        # run a "apt-get update" now, its ok to ignore errors, 
        # because 
        # a) we disable any third party sources later
        # b) we check if we have valid ubuntu sources later
        #    after we rewrite the sources.list and do a 
        #    apt-get update there too
        # because the (unmodified) sources.list of the user
        # may contain bad/unreachable entries we run only
        # with a single retry
        self.doUpdate(showErrors=False, forceRetries=1)
        self.openCache()

        # do pre-upgrade stuff (calc list of obsolete pkgs etc)
        if not self.doPostInitialUpdate():
            self.abort()

        # update sources.list
        self._view.setStep(STEP_MODIFY_SOURCES)
        self._view.updateStatus(_("Updating repository information"))
        if not self.updateSourcesList():
            self.abort()

        # add cdrom (if we have one)
        if (self.aptcdrom and
            not self.aptcdrom.add(self.sources_backup_ext)):
            self._view.error(_("Failed to add the cdrom"),
                             _("Sorry, adding the cdrom was not successful."))
            sys.exit(1)

        # then update the package index files
        if not self.doUpdate():
            self.abort()

        # then open the cache (again)
        self._view.updateStatus(_("Checking package manager"))
        # if something fails here (e.g. locking the cache) we need to
        # restore the system state (LP: #1052605)
        self.openCache(restore_sources_list_on_fail=True)

        # re-check server mode because we got new packages (it may happen
        # that the system had no sources.list entries and therefore no
        # desktop file information)
        self.serverMode = self.cache.need_server_mode()
        # do it here as we neeed to know if we are in server or client mode
        self.quirks.ensure_recommends_are_installed_on_desktops()
        # now check if we still have some key packages available/downloadable
        # after the update - if not something went seriously wrong
        # (this happend e.g. during the intrepid->jaunty upgrade for some
        #  users when de.archive.ubuntu.com was overloaded)
        for pkg in self.config.getlist("Distro","BaseMetaPkgs"):
            if (pkg not in self.cache or
                not self.cache.anyVersionDownloadable(self.cache[pkg])):
                # FIXME: we could offer to add default source entries here,
                #        but we need to be careful to not duplicate them
                #        (i.e. the error here could be something else than
                #        missing sources entries but network errors etc)
                logging.error("No '%s' available/downloadable after sources.list rewrite+update" % pkg)
                self._view.error(_("Invalid package information"),
                                 _("After updating your package "
                                   "information, the essential package '%s' "
                                   "could not be located. This may be "
                                   "because you have no official mirrors "
                                   "listed in your software sources, or "
                                   "because of excessive load on the mirror "
                                   "you are using. See /etc/apt/sources.list "
                                   "for the current list of configured "
                                   "software sources."
                                   "\n"
                                   "In the case of an overloaded mirror, you "
                                   "may want to try the upgrade again later.")
                                   % pkg)
                subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
                self.abort()

        # calc the dist-upgrade and see if the removals are ok/expected
        # do the dist-upgrade
        self._view.updateStatus(_("Calculating the changes"))
        if not self.askDistUpgrade():
            self.abort()

        # fetch the stuff
        self._view.setStep(STEP_FETCH)
        self._view.updateStatus(_("Fetching"))
        if not self.doDistUpgradeFetching():
            self.abort()

        # now do the upgrade
        self._view.setStep(STEP_INSTALL)
        self._view.updateStatus(_("Upgrading"))
        if not self.doDistUpgrade():
            # run the post install scripts (for stuff like UUID conversion)
            self.runPostInstallScripts()
            # don't abort here, because it would restore the sources.list
            self._view.information(_("Upgrade complete"),
                                   _("The upgrade has completed but there "
                                     "were errors during the upgrade "
                                     "process."))
            sys.exit(1) 
            
        # do post-upgrade stuff
        self._view.setStep(STEP_CLEANUP)
        self._view.updateStatus(_("Searching for obsolete software"))
        self.doPostUpgrade()

        # comment out cdrom source
        if self.aptcdrom:
            self.aptcdrom.comment_out_cdrom_entry()

        # done, ask for reboot
        self._view.setStep(STEP_REBOOT)
        self._view.updateStatus(_("System upgrade is complete."))            
        # FIXME should we look into /var/run/reboot-required here?
        if (not inside_chroot() and
            self._view.confirmRestart()):
            subprocess.Popen("/sbin/reboot")
            sys.exit(0)
        return True
        
    def run(self):
        self._view.processEvents()
        return self.fullUpgrade()
    
    def doPartialUpgrade(self):
        " partial upgrade mode, useful for repairing "
        self._view.setStep(STEP_PREPARE)
        self._view.hideStep(STEP_MODIFY_SOURCES)
        self._view.hideStep(STEP_REBOOT)
        self._partialUpgrade = True
        self.prepare()
        if not self.doPostInitialUpdate():
            return False
        if not self.askDistUpgrade():
            return False
        self._view.setStep(STEP_FETCH)
        self._view.updateStatus(_("Fetching"))
        if not self.doDistUpgradeFetching():
            return False
        self._view.setStep(STEP_INSTALL)
        self._view.updateStatus(_("Upgrading"))
        if not self.doDistUpgrade():
            self._view.information(_("Upgrade complete"),
                                   _("The upgrade has completed but there "
                                     "were errors during the upgrade "
                                     "process."))
            return False
        self._view.setStep(STEP_CLEANUP)
        if not self.doPostUpgrade():
            self._view.information(_("Upgrade complete"),
                                   _("The upgrade has completed but there "
                                     "were errors during the upgrade "
                                     "process."))
            return False

        if os.path.exists(REBOOT_REQUIRED_FILE):
            # we can not talk to session management here, we run as root
            if self._view.confirmRestart():
                subprocess.Popen("/sbin/reboot")
        else:
            self._view.information(_("Upgrade complete"),
                                   _("The partial upgrade was completed."))
        return True


if __name__ == "__main__":
    from .DistUpgradeViewText import DistUpgradeViewText
    logging.basicConfig(level=logging.DEBUG)
    v = DistUpgradeViewText()
    dc = DistUpgradeController(v)
    #dc.openCache()
    dc._disableAptCronJob()
    dc._enableAptCronJob()
    #dc._addRelatimeToFstab()
    #dc.prepare()
    #dc.askDistUpgrade()
    #dc._checkFreeSpace()
    #dc._rewriteFstab()
    #dc._checkAdminGroup()
    #dc._rewriteAptPeriodic(2)