~urbanape/bindwood/subsequent-client

/*
 * Copyright 2009 Canonical Ltd.
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 3, as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranties of
 * MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
 * PURPOSE.  See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
/* Lots and lots of debugging information */

Components.utils.import("resource://gre/modules/utils.js");
Components.utils.import("resource://bindwood/couch.jsm");

var EXPORTED_SYMBOLS = ["Bindwood"];

var Cc = Components.classes;
var Ci = Components.interfaces;

var bookmarksService = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]
    .getService(Ci.nsINavBookmarksService);
var livemarkService = Cc["@mozilla.org/browser/livemark-service;2"]
    .getService(Ci.nsILivemarkService);
var uuidService = Cc["@mozilla.org/uuid-generator;1"]
    .getService(Ci.nsIUUIDGenerator);
var annotationService = Cc["@mozilla.org/browser/annotation-service;1"]
    .getService(Ci.nsIAnnotationService);
var consoleService = Cc["@mozilla.org/consoleservice;1"]
    .getService(Ci.nsIConsoleService);
var historyService = Cc["@mozilla.org/browser/nav-history-service;1"]
    .getService(Ci.nsINavHistoryService);
var ioService = Cc["@mozilla.org/network/io-service;1"]
    .getService(Ci.nsIIOService);
var envService = Cc["@mozilla.org/process/environment;1"]
    .getService(Ci.nsIEnvironment);
var directoryService = Cc["@mozilla.org/file/directory_service;1"]
    .getService(Ci.nsIProperties);
var windowService = Cc["@mozilla.org/appshell/window-mediator;1"]
    .getService(Ci.nsIWindowMediator);
// Technically, a branch, rather than the actual service, but
// consistency wins, I think
var prefsService = Cc["@mozilla.org/preferences-service;1"]
    .getService(Ci.nsIPrefService).getBranch("bindwood.");

var Bindwood = {

    timer: Cc["@mozilla.org/timer;1"]
        .createInstance(Ci.nsITimer),

    TYPE_BOOKMARK: "http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark",
    TYPE_FOLDER: "http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder",
    TYPE_FEED: "http://www.freedesktop.org/wiki/Specifications/desktopcouch/feed",
    TYPE_SEPARATOR: "http://www.freedesktop.org/wiki/Specifications/desktopcouch/separator",

    running: false,
    push: 'ENABLED', // Start off enabled
    annotationKey: "bindwood/uuid",
    uuidItemIdMap: {},
    records: [],
    seen_revisions: {},
    timestamps: {},

    // Debugging/Console Behavior
    writeMessage: function(aMessage) {
        // convenience method for logging. Way better than alert()s.
        if (prefsService.getBoolPref("debug")) {
            consoleService.logStringMessage(
                "Bindwood: " + aMessage);
        }
    },

    writeError: function(aMessage, e) {
        // This should fire whether we're in DEBUG or not
        consoleService.logStringMessage(
            "Bindwood: " + aMessage +
                " message: '" + e.message +
                "', reason: '" + e.reason +
                "', description: '" + e.description +
                "', error: '" + e.error +
                "', raw error: '" + JSON.stringify(e) + "'");
    },

    noteStartTime: function(key) {
        var now = new Date();
        Bindwood.timestamps[key] = now.getTime();
    },

    noteEndTime: function(key) {
        var now = new Date();
        var diff = now.getTime() - Bindwood.timestamps[key];
        Bindwood.writeMessage(
            key + " took " + diff + " milliseconds");
        delete Bindwood.timestamps[key];
    },

    extractProfileName: function(path) {
        // We want the last part of the profile path
        // ('default' for '/home/zbir/.mozilla/firefox/ixlw0nvl.default')
        // For profiles that have not been created via the Profile Manager,
        // we just take the last path segment as the profile name.
        //
        // Actually, there's a degenerate case, where the last segment
        // doesn't have the Firefox-provided random string:
        //   '/home/zbir/canonical.com' => 'com'
        // But as I said, this is maybe degenerate.

        var segments = path.split('/');
        var possible_profile = segments[segments.length - 1];
        var first_period = possible_profile.indexOf('.');
        if (first_period == -1) {
            // no periods in the last segment, return as is
            return possible_profile;
        } else {
            return possible_profile.substr(first_period + 1);
        }
    },

    // Setup the environment and go
    init: function() {
        // Start the process and de-register ourself
        // http://forums.mozillazine.org/viewtopic.php?f=19&t=657911&start=0
        // It ensures that we're only running that code on the first window.

        if (!Bindwood.running) {
            Bindwood.writeMessage("Getting started in init.");
            Bindwood.currentProfile = Bindwood.extractProfileName(
                directoryService.get('ProfD', Ci.nsIFile).path);

            Bindwood.writeMessage("Got our profile: " + Bindwood.currentProfile);

            Bindwood.running = true;
            bookmarksService.addObserver(Bindwood.Observer, false);
            Bindwood.getCouchEnvironment();
        }
    },

    getCouchEnvironment: function() {
        // find the desktop Couch port number by making a D-Bus call
        // we call D-Bus by shelling out to a bash script which calls
        // it for us, and writes the port number into a temp file

        // find OS temp dir to put the tempfile in
        // https://developer.mozilla.org/index.php?title=File_I%2F%2FO#Getting_special_files
        var tmpdir = Cc["@mozilla.org/file/directory_service;1"]
            .getService(Ci.nsIProperties).get("TmpD", Ci.nsIFile);
        // create a randomly named tempfile in the tempdir
        var tmpfile = Cc["@mozilla.org/file/local;1"]
            .createInstance(Ci.nsILocalFile);
        tmpfile.initWithPath(tmpdir.path + "/desktopcouch." + Math.random());
        tmpfile.createUnique(tmpfile.NORMAL_FILE_TYPE, 0600);

        // find the D-Bus bash script, which is in our extension folder
        var MY_ID = "bindwood@ubuntu.com";
        var em = Cc["@mozilla.org/extensions/manager;1"]
            .getService(Ci.nsIExtensionManager);
        var couchdb_env_script = em.getInstallLocation(MY_ID)
            .getItemFile(MY_ID, "couchdb_env.sh");
        // create an nsILocalFile for the executable
        var nsifile = Cc["@mozilla.org/file/local;1"]
            .createInstance(Ci.nsILocalFile);
        nsifile.initWithPath(couchdb_env_script.path);

        // create an nsIProcess2 to execute this bash script
        var process = Cc["@mozilla.org/process/util;1"]
            .createInstance(Ci.nsIProcess2 || Ci.nsIProcess);
        process.init(nsifile);

        // Run the process, passing the tmpfile path
        var args = [tmpfile.path];
        process.runAsync(args, args.length, {
            observe: function(process, finishState, unused_data) {
                // If the script exists cleanly, we should have a file
                // containing the port couch is running on as well as
                // the various OAuth tokens necessary to talk to it.
                var shouldProceed = true;
                if (finishState == "process-finished") {
                    // read temp file to find couch environment
                    // https://developer.mozilla.org/en/Code_snippets/File_I%2f%2fO#Reading_from_a_file
                    var environment;
                    var fstream = Cc["@mozilla.org/network/file-input-stream;1"]
                        .createInstance(Ci.nsIFileInputStream);
                    var cstream = Cc["@mozilla.org/intl/converter-input-stream;1"]
                        .createInstance(Ci.nsIConverterInputStream);
                    fstream.init(tmpfile, -1, 0, 0);
                    cstream.init(fstream, "UTF-8", 0, 0);
                    let (str = {}) {
                        // read the whole file and put it in str.value
                        cstream.readString(-1, str);
                        environment = str.value;
                    };
                    cstream.close(); // this closes fstream
                    environment = environment.replace(/^\s\s*/, '')
                        .replace(/\s\s*$/, '');
                } else {
                    // If we fail, we should just return
                    Bindwood.writeMessage("D-Bus port find failed");
                    shouldProceed = false;
                }
                tmpfile.remove(false);

                if (environment == 'ENOCOUCH') {
                    // No Couch environment found. Just spit out a
                    // message and return, stopping Bindwood from
                    // doing anything further.
                    Bindwood.writeError(
                        "No suitable Couch environment found." +
                            " Not proceeding.", e);
                    shouldProceed = false;
                }

                if (shouldProceed && environment) {
                    Bindwood.writeMessage("Got our environment, proceeding.");
                    Bindwood.setUpEnvironment(environment);
                } else {
                    // Unregister our observer for bookmark events; we're done
                    Bindwood.writeMessage("No environment. Unregistering observer.");
                    bookmarksService.removeObserver(Bindwood.Observer);
                }
            }
        });
    },

    setUpEnvironment: function(couchEnvironment) {
        var env_array = couchEnvironment.split(':');
        var port = env_array[0];
        var consumer_key = env_array[1];
        var consumer_secret = env_array[2];
        var token = env_array[3];
        var token_secret = env_array[4];

        CouchDB.port = port;
        CouchDB.accessor = {
            consumerSecret: consumer_secret,
            tokenSecret: token_secret
        };
        CouchDB.message = {
            parameters: {
                oauth_callback: "None",
                oauth_consumer_key: consumer_key,
                oauth_signature_method: "PLAINTEXT",
                oauth_token: token,
                oauth_verifier: "None",
                oauth_version: "1.0"
            }
        };

        var db_name = 'bookmarks';
        if (envService.exists('BINDWOOD_DB')) {
            db_name = envService.get('BINDWOOD_DB');
        }

        Bindwood.writeMessage("Got our db name: " + db_name);
        Bindwood.couch = new CouchDB(db_name);

        try {
            Bindwood.startProcess();
        } catch(e) {
            Bindwood.writeError(
                "Something wrong with the process, exiting.", e);
            return;
        }
    },

    getLastSequence: function() {
        var seq;
        try {
            seq = prefsService.getIntPref('last_seq');
        } catch(e) {
            seq = 0;
        }
        return seq;
    },

    setLastSequence: function(seq) {
        prefsService.setIntPref('last_seq', seq);
        return seq;
    },

    getLatestModified: function() {
        var mod;
        try {
            mod = Number(prefsService.getCharPref('latest_modified'));
        } catch(e) {
            mod = 0;
        }
        return mod;
    },

    setLatestModified: function(mod) {
        prefsService.setCharPref(
            'latest_modified', Number(mod).toString());
        return mod;
    },

    startProcess: function() {
        Bindwood.writeMessage("Starting process");
        Bindwood.last_seq = Bindwood.getLastSequence();
        Bindwood.writeMessage("Got our last known sequence number: " + Bindwood.last_seq);
        Bindwood.latest_modified = Bindwood.getLatestModified();
        Bindwood.writeMessage("Got our latest known last_modified: " + Bindwood.latest_modified);

        Bindwood.writeMessage("Ensuring the database exisits");
        Bindwood.ensureDatabase();
        Bindwood.writeMessage("Ensuring the views exist");
        Bindwood.ensureViews();
        Bindwood.writeMessage("Ensuring our scratch folder exists");
        Bindwood.scratch_folder = Bindwood.ensureLocalScratchFolder();

        var HAVE_LAST_SEQ = Bindwood.last_seq ? 1 : 0; // 0 or 1
        var HAVE_PROFILE_ROOT = Bindwood.profileExists(); // 0 or 2

        var after_function = function() {};

        switch(HAVE_LAST_SEQ | HAVE_PROFILE_ROOT) {
        case 0:
            // Neither the profile root exists, nor do we have a last_seq. Ergo, we are a first
            // time user. Proceed normally.

            // Bindwood.display_first_sync_page();
            break;
        case 1:
            // We have a last_seq, but the profile root does not exist. Ergo, we are an old user
            // and must migrate to the new way of doing things.

            /* Migration strategy:

               Pull all records from Couch, and for each:

                 - get the uuid off the bookmark.

                 - look up itemId by uuid

                 - annotate the itemId with the document's _id

                 - delete the uuid field off the record
            
             */

            // Bindwood.display_migration_sync_page();
            break;
        case 2:
            // We have no last_seq, but the profile root exists. Ergo, we are starting up a
            // subsequent client. We must make our local bookmarks look like remote, and ensure
            // that any unaccounted for local bookmarks are sent to CouchDB.

            var additional = Bindwood.handleSubsequentClientFirstTimeSync();

            after_function = function() {
                for (var i = 0; i < additional.length; i++) {
                    try {
                        var response = Bindwood.couch.save(additional[i]);
                        // We can avoid having to process this revision when we pull it later
                        Bindwood.seen_revisions[response.rev] = true;
                    } catch(e) {
                        Bindwood.writeError(
                            "Problem saving record to CouchDB; record is " +
                                JSON.stringify(record) + ": ", e);
                    }
                }
            };
            break;
        case 3: // Should this just be default?
            // We have a last_seq, and the profile root exists. Ergo, we are a normally operating
            // client. Proceed normally.
            break;
        default:
            break;
        }

        Bindwood.noteStartTime('Generating the manifest');
        Bindwood.generateManifest();
        Bindwood.noteEndTime('Generating the manifest');
        Bindwood.noteStartTime('Pushing records');
        Bindwood.pushLatestRecords();
        Bindwood.noteEndTime('Pushing records');

        after_function();

        try {
            Bindwood.pullChanges();
        } catch(e) {
            Bindwood.writeError("Problem pulling changes!", e);
        }
    },

    ensureDatabase: function() {
        // This function will create the database if it does not
        // exist, but we return a boolean representing whether or not
        // the database existed prior.
        try {
            Bindwood.couch.createDb();
        } catch (e) {
            if (e.error != 'file_exists') {
                Bindwood.writeError("Error creating database: ", e);
                throw(e);
            }
        }
    },

    ensureViews: function() {
        var view = {
            _id: "_design/bookmarks",
            views: {
                profile: {
                    map: "function(doc) { var scheme = doc.uri.split(':',1)[0]; var uri; if (scheme == 'http' || scheme == 'https') {uri = doc.uri.split('/')[2];if (uri.length < 30) { uri += '/' + doc.uri.split('/',4)[3].substr(0,30-uri.length) + '...';}} else {uri = scheme + ' URL';}if ((!(doc.application_annotations && doc.application_annotations['Ubuntu One'] && doc.application_annotations['Ubuntu One'].private_annotations && doc.application_annotations['Ubuntu One'].private_annotations.deleted)) && (doc.application_annotations.Firefox.profile)) {emit(doc.application_annotations.Firefox.profile, [doc.title, uri]);}}"
                },
                live_bookmarks: {
                    map: "function(doc) { var scheme = doc.uri.split(':',1)[0]; var uri; if (scheme == 'http' || scheme == 'https') {uri = doc.uri.split('/')[2];if (uri.length < 30) { uri += '/' + doc.uri.split('/',4)[3].substr(0,30-uri.length) + '...';}} else {uri = scheme + ' URL';}if (!(doc.application_annotations && doc.application_annotations['Ubuntu One'] && doc.application_annotations['Ubuntu One'].private_annotations && doc.application_annotations['Ubuntu One'].private_annotations.deleted)) {emit(doc.title, uri);}}"
                },
                deleted_bookmarks: {
                    map: "function(doc) { if (doc.application_annotations && doc.application_annotations['Ubuntu One'] && doc.application_annotations['Ubuntu One'].private_annotations && doc.application_annotations['Ubuntu One'].private_annotations.deleted) { emit (doc.title, doc.uri); } }"
                }
            }
        };

        try {
            var doc = Bindwood.couch.open(view._id);
            if (!doc) {
                doc = view;
            }
            try {
                Bindwood.couch.save(doc);
            } catch(e) {
                Bindwood.writeError("Problem saving view: ", e);
            }
        } catch(e) {
            // some kind of error fetching the existing design doc
            Bindwood.writeError("Problem checking for view: ", e);
        }
    },

    ensureLocalScratchFolder: function() {
        // Because records come from CouchDB without a parent field, until
        // we get an updated folder record with a record situated properly
        // in its children field, we need a place to temporarily store
        // records from Couch. This is that place.
        var folder = bookmarksService.unfiledBookmarksFolder;
        var rootNode = Bindwood.getFolderRoot(folder);
        rootNode.containerOpen = true;
        for (var i=0; i<rootNode.childCount; i++) {
            var node = rootNode.getChild(i);
            if (node.title == 'Desktop Couch Scratch') {
                rootNode.containerOpen = false;
                return node.itemId;
            }
        }
        rootNode.containerOpen = false;

        var folderId = bookmarksService.createFolder(
            folder, 'Desktop Couch Scratch', -1);
        return folderId;
    },

    profileExists: function() {
        // Check to see if the current profile's manifest exists in
        // the database.
        try {
            Bindwood.couch.open('root_' + Bindwood.currentProfile);
        } catch(e) {
            if (e.error == 'not_found') {
                return 0;
            }
        }
        return 2;
    },

    handleSubsequentClientFirstTimeSync: function() {
        // Bindwood.display_subsequent_client_sync_page();
        // Show a helpful page explaining what's happening, so the user doesn't feel
        // something is broken.

        Bindwood.writeMessage("We're a subsequent client. Let's merge the remote and the local.");


        // get the remote toolbarFolder from Couch.
        var remote_root = Bindwood.couch.open('root_' + Bindwood.currentProfile);
        var local_roots = [bookmarksService.toolbarFolder,
                           bookmarksService.bookmarksMenuFolder,
                           bookmarksService.unfiledBookmarksFolder];

        var records_needing_pushing = [];

        for (var i = 0; i < local_roots.length; i++) {
            Bindwood.syncRemoteAndLocal(remote_root.children[i], local_roots[i], records_needing_pushing);
        }

        return records_needing_pushing;
    },

    syncRemoteAndLocal: function(remote_folder, local_folder, accum) {
        var push_local = false;

        // start with a folder like toolbarFolder
        var local = Bindwood.getFolderRoot(local_folder);
        local.containerOpen = true;

        // walk the remote folder's children, and ask for each one:
        var remote = Bindwood.couch.open(remote_folder);
        var remote_children = remote.children;
        Bindwood.writeMessage("Beginning to walk remote children");
        for (var i = 0; i < remote_children.length; i++) {
            var remote_child = Bindwood.couch.open(remote_children[i]);
            var local_child;
            var found_local = false;

            Bindwood.writeMessage("Looking for record type: " + remote_child.record_type +
                " identified by " + remote_child.record_type != Bindwood.TYPE_SEPARATOR ? remote_child.title : "being a separator");

            // does my local toolbarFolder have this child anywhere in it?
            for (var j = 0; j < local.childCount; j++) {
                local_child = local.getChild(j);

                // Check to see whether we're testing a separator (which has no title)
                // or whether we're testing the same type and title
                if ((Bindwood.sameType(local_child, remote_child) &&
                     remote_child.record_type == Bindwood.TYPE_SEPARATOR) ||
                    (Bindwood.sameType(local_child, remote_child) &&
                     Bindwood.sameTitle(local_child, remote_child))) {
                    found_local = true;
                    Bindwood.writeMessage("Found the record.");
                    Bindwood.annotateItemWithUUID(local_child.itemId, remote_child._id);
                    // If we're dealing with a folder, we'll process it recursively,
                    // and the only other thing we'd impose would be the title,
                    // which already matches..
                    if (remote_child.record_type != Bindwood.TYPE_FOLDER) {
                        Bindwood.processCouchRecord(remote_child);
                    }
                    if (i != j) { // if yes, but in a different location
                        // Move the local to the proper index within the same folder.
                        Bindwood.writeMessage("Record isn't in the same location as remote, moving it.");
                        Bindwood.makeLocalChangeOnly(
                            function() {
                                bookmarksService.moveItem(
                                    local_child.itemId, local_folder, i);
                            }
                        );
                        local_needs_pushing = true;
                    }
                }
            }

            if (!found_local) {
                // Add the record locally, annotate it, and place it in the correct index.
                // Add current local folder as one that needs to be pushed back (changing its children)

                Bindwood.writeMessage("Remote record doesn't exist here, recreating it.");
                Bindwood.processCouchRecord(remote_child, local_folder, i);
                local_needs_pushing = true;
            }

            // is the child a folder?
            if (remote_child.record_type == Bindwood.TYPE_FOLDER) {
                // Recurse into the function with remote id and local folder id
                Bindwood.syncRemoteAndLocal(remote_child._id, local_child, accum);
            }
        }

        local.containerOpen = false;

        if (local_needs_pushing) {
            accum.push(Bindwood.couchRecordForItemId(local_folder));
        }
    },

    sameType: function(localNode, remoteDoc) {
        switch(remoteDoc.record_type) {
        case Bindwood.TYPE_BOOKMARK:
            return PlacesUtils.nodeIsBookmark(localNode);
        case Bindwood.TYPE_FEED:
            return PlacesUtils.nodeIsLivemarkContainer(localNode);
        case Bindwood.TYPE_FOLDER:
            return PlacesUtils.nodeIsFolder(localNode);
        case Bindwood.TYPE_SEPARATOR:
            return PlacesUtils.nodeIsSeparator(localNode);
        default:
            return false;
        }
    },

    sameTitle: function(localNode, remoteDoc) {
        return localNode.title == remoteDoc.title;
    },

    // Looking up records locally
    annotateItemWithUUID: function(itemId, seed_uuid) {
        var uuid = (seed_uuid ?
                    seed_uuid :
                    uuidService.generateUUID().toString());
        Bindwood.writeMessage("UUID We came up with: " + uuid);
        Bindwood.writeMessage("Annotating the item now.");
        annotationService.setItemAnnotation(
            itemId,
            Bindwood.annotationKey,
            uuid,
            0,
            annotationService.EXPIRE_NEVER);
        // Whenever we create a new UUID, stash it and the itemId in
        // our local cache.
        Bindwood.uuidItemIdMap[uuid] = itemId;
        return uuid;
    },

    itemIdForUUID: function(uuid) {
        // First, try to look it up in our local cache, barring that
        // (which shouldn't happen), look it up slowly.
        var itemId = Bindwood.uuidItemIdMap[uuid];

        if (!itemId) {
            var items = annotationService.getItemsWithAnnotation(
                Bindwood.annotationKey, {});
            var num_items = items.length;
            for (var i = 0; i < items.length; i++) {
                if (annotationService.getItemAnnotation(
                        items[i], Bindwood.annotationKey) == uuid) {
                    Bindwood.uuidItemIdMap[uuid] = itemId = items[i];
                    break;
                }
            }
            if (!itemId) {
                Bindwood.writeMessage(
                    "XXX: Still haven't found the right itemId!");
            }
        }
        return itemId;
    },

    uuidForItemId: function(itemId) {
        // Try to look up the uuid, and failing that, assign a new one
        // and return it.
        var uuid;
        try {
            uuid = annotationService.getItemAnnotation(
                itemId, Bindwood.annotationKey);
            Bindwood.uuidItemIdMap[uuid] = itemId;
        } catch(e) {
            Bindwood.writeError(
                "Couldn't find a UUID for itemId: " + itemId, e);
            uuid = Bindwood.makeLocalChangeOnly(
                function() { return Bindwood.annotateItemWithUUID(
                                 itemId, null); } );
        }

        return uuid;
    },

    couchRecordForItemId: function(itemId) {
        var bs = bookmarksService;

        var uuid = Bindwood.uuidForItemId(itemId);
        var profile = Bindwood.currentProfile;
        var last_modified = bookmarksService.getItemLastModified(itemId);

        var record = {
            "_id": uuid,
            application_annotations: {
                Firefox: {
                    profile: profile,
                    last_modified: last_modified
                }
            }
        };

        switch(bs.getItemType(itemId)) {
        case bs.TYPE_BOOKMARK:
            record.title = bs.getItemTitle(itemId);
            record.record_type = Bindwood.TYPE_BOOKMARK;
            record.uri = bs.getBookmarkURI(itemId).spec;
            break;
        case bs.TYPE_FOLDER:
            record.title = bs.getItemTitle(itemId);

            // Firefox doesn't differentiate between regular folders
            // and livemark folders. *sigh* So, we override it here
            if (livemarkService.isLivemark(itemId)) {
                record.record_type = Bindwood.TYPE_FEED;
                record.site_uri = livemarkService.getSiteURI(itemId).spec;
                record.feed_uri = livemarkService.getFeedURI(itemId).spec;
            } else {
                record.record_type = Bindwood.TYPE_FOLDER;
                record.children = [];
            }
            break;
        case bs.TYPE_SEPARATOR:
            record.record_type = Bindwood.TYPE_SEPARATOR;
            break;
        default:
            break;
        }

        return record;
    },

    makeLocalChangeOnly: function(func) {
        Bindwood.push = 'DISABLED';
        var results = func();
        Bindwood.push = 'ENABLED';
        return results;
    },

    // Back and forth
    getFolderRoot: function(folder) {
        var options = historyService.getNewQueryOptions();
        var query = historyService.getNewQuery();
        query.setFolders([folder], 1);
        var result = historyService.executeQuery(query, options);
        return result.root;
    },

    getUUIDsFromFolder: function(folder) {
        var folderRoot = Bindwood.getFolderRoot(folder);
        folderRoot.containerOpen = true;
        var uuids = [];

        for (var i=0; i<folderRoot.childCount; i++) {
            var node = folderRoot.getChild(i);
            uuids.push(Bindwood.uuidForItemId(node.itemId));
        }

        return uuids;
    },

    getRecordsFromFolder: function(folder) {
        // Make a record for us, populating a children field with the _ids of all our children
        var folderRoot = Bindwood.getFolderRoot(folder);
        folderRoot.containerOpen = true;

        var folder_record = Bindwood.couchRecordForItemId(folder);

        for (var i=0; i<folderRoot.childCount; i++) {
            var node = folderRoot.getChild(i);

            var record = Bindwood.couchRecordForItemId(node.itemId);
            folder_record.children.push(record._id);

            // If node is a folder (but not a Livemark or Dynamic container),
            // descend into it, looking for its contents
            if (record.record_type == Bindwood.TYPE_FOLDER) {
                Bindwood.getRecordsFromFolder(node.itemId)
            } else {
                Bindwood.records.push(record);
            }
        }
        folderRoot.containerOpen = false;
        Bindwood.records.push(folder_record);
        return folder_record;
    },

    generateManifest: function() {
        // Fill up the Bindwood.manifest and initial push lists
        var primaryFolders = [bookmarksService.toolbarFolder,
                              bookmarksService.bookmarksMenuFolder,
                              bookmarksService.unfiledBookmarksFolder];

        var profile_root = {
            "_id": "root_" + Bindwood.currentProfile,
            children: [],
            application_annotations: {
                Firefox: {
                    profile: Bindwood.currentProfile,
                    last_modified: 1
                }
            }
        };

        for (var i=0; i<primaryFolders.length; i++) {
            var folder = primaryFolders[i];
            var folder_record = Bindwood.getRecordsFromFolder(folder);

            profile_root.children.push(folder_record._id);
        }

        Bindwood.records.push(profile_root);
    },

    sortByLastModDesc: function(a, b) {
        var a_mod = a.application_annotations.Firefox.last_modified;
        var b_mod = b.application_annotations.Firefox.last_modified;
        return b_mod - a_mod; // descending
    },

    pushLatestRecords: function() {
        Bindwood.records.sort(Bindwood.sortByLastModDesc);
        // Now that the record are all sorted descending by last
        // mod time, we can check each in turn to see if its mod time
        // is greater than our persisted mod time. Afterwards, we'll
        // set our persisted latest mod time to be the first record's
        // mod time.
        var newest = Bindwood.records[0];
        var newest_ff = newest.application_annotations.Firefox;
        var new_latest_modified = newest_ff.last_modified;
        for (var i = 0; i < Bindwood.records.length; i++) {
            // find this record in CouchDB
            var record = Bindwood.records[i];
            var ff = record.application_annotations.Firefox;

            if (ff.last_modified <= Bindwood.latest_modified) {
                Bindwood.writeMessage(
                    "We've reached records we've already dealt with." +
                        " Breaking out of the loop.");
                break;
            }

            var doc = Bindwood.couch.open(record._id);
            if (!doc) {
                // this record is not in CouchDB, so write it
                try {
                    var response = Bindwood.couch.save(record);
                    // We can avoid having to process this revision when we pull it later
                    Bindwood.seen_revisions[response.rev] = true;
                } catch(e) {
                    Bindwood.writeError(
                        "Problem saving record to CouchDB; record is " +
                            JSON.stringify(record) + ": ", e);
                }
            } else {
                // record is already in CouchDB, so do nothing
                Bindwood.writeMessage(
                    "This record (" + record._id + ") is already in Couch, skipping");
            }
        }
        Bindwood.latest_modified = Bindwood.setLatestModified(
            new_latest_modified);
    },

    pushFolderChildren: function(folder, children) {
        var doc = Bindwood.couch.open(Bindwood.uuidForItemId(folder));
        var new_children = children;
        if (!new_children) {
            new_children = Bindwood.getUUIDsFromFolder(folder);
        }
        doc.children = new_children;
        var response = Bindwood.couch.save(doc);
        Bindwood.seen_revisions[response.rev] = true;
    },

    pullChanges: function() {
        var repeater = {
            notify: function(timer) {
                Bindwood.pullRecords();
                Bindwood.writeMessage(
                    "Successful run, rescheduling ourself");
            }
        };

        // reschedule ourself
        try {
            Bindwood.timer.initWithCallback(repeater, 30000, Ci.nsITimer.TYPE_REPEATING_SLACK);
        } catch(e) {
            Bindwood.writeError("Problem setting up repeater.", e);
        }
    },

    pullRecords: function() {
        // Check to see if our prefsService has a preference set (I
        // know, bad form) for last_seq, which would designate the
        // last Couch sequence we've seen (this might be 0 if we've
        // never synced before, in which case, we'd get all
        // changes). Afterwards, set the last known sequence in
        // prefs. Then, future polls will use last_seq as the start
        // for finding changes.

        // XXX: currently, we do a single changes pull. Eventually, if
        // we need to use threads, we can do long polling in a
        // background thread.

        Bindwood.noteStartTime('Pulling records');
        var results = {results: [], last_seq: 0};
        try {
            results = Bindwood.couch.changes(
                {since: Bindwood.last_seq},
                null
            );
        } catch(e) {
            Bindwood.writeError(
                "Problem long polling bookmarks from Couch: ", e);
        }
        var revisions = results.results;
        for (var i = 0; i < revisions.length; i++) {
            var rev = revisions[i];
            var revno = rev.changes[0].rev;
            var recordid = rev.id;

            // Skip (for now) if we're dealing with a root folder or a
            // design doc
            if (recordid.indexOf('root_') === 0 ||
                recordid.indexOf('_design') === 0) {
                Bindwood.writeMessage("Root profile or design doc, skipping...");
                continue;
            }

            // Skip any revisions we've already seen (because we just
            // put them there)
            if (Bindwood.seen_revisions[revno]) {
                Bindwood.writeMessage("We've seen this revision (" + revno + ") before, when we created it.");
                delete Bindwood.seen_revisions[revno];
                continue;
            }

            var record = Bindwood.couch.open(recordid);

            if (!Bindwood.recordInCurrentProfile(record)) {
                Bindwood.writeMessage("Record isn't in our current profile. Skipping...");
                continue;
            }

            // Next, check to see if the record we've pulled down
            // is flagged as deleted.  If so, we should make sure any
            // local copy we have of this record has also been
            // deleted.
            if (Bindwood.isDeleted(record)) {
                Bindwood.makeLocalChangeOnly(
                    function() {
                        Bindwood.writeMessage(
                            "Record in Couch marked as deleted;" +
                                " attempting to delete local copy.");
                        Bindwood.deleteLocalRecord(record);
                    });
                continue; // Don't bother continuing to process anything further in this revision
            }

            Bindwood.processCouchRecord(record);
        }

        Bindwood.last_seq = Bindwood.setLastSequence(results.last_seq);
        Bindwood.noteEndTime('Pulling records');
    },

    recordInCurrentProfile: function(record) {
        if (record.application_annotations &&
            record.application_annotations.Firefox &&
            record.application_annotations.Firefox.profile &&
            record.application_annotations.Firefox.profile == Bindwood.currentProfile) {
            return true;
        }
        return false;
    },

    isDeleted: function(record) {
        if (record.application_annotations &&
            record.application_annotations["Ubuntu One"] &&
            record.application_annotations["Ubuntu One"].private_application_annotations &&
            record.application_annotations["Ubuntu One"].private_application_annotations.deleted) {
            return true;
        }
        return false;
    },

    deleteLocalRecord: function(record) {
        // If we can't resolve the itemId, even by looking up URI,
        // assume it's already gone.
        var itemId = Bindwood.itemIdForUUID(record._id);
        if (itemId) {
            return bookmarksService.removeItem(itemId);
        }
    },

    processCouchRecord: function(record, aParent, aIndex) {
        var aParent = aParent ? aParent : Bindwood.scratchFolder;
        var aIndex = aIndex ? aIndex : -1;

        switch(record.record_type) {
        case Bindwood.TYPE_BOOKMARK:
            Bindwood.makeLocalChangeOnly(
                function() {
                    Bindwood.processCouchBookmarkRevision(record, aParent, aIndex);
                });
            break;
        case Bindwood.TYPE_FOLDER:
            Bindwood.makeLocalChangeOnly(
                function() {
                    Bindwood.processCouchFolderRevision(record, aParent, aIndex);
                });
            break;
        case Bindwood.TYPE_FEED:
            Bindwood.makeLocalChangeOnly(
                function() {
                    Bindwood.processCouchFeedRevision(record, aParent, aIndex);
                });
            break;
        case Bindwood.TYPE_SEPARATOR:
            Bindwood.makeLocalChangeOnly(
                function() {
                    Bindwood.processCouchSeparatorRevision(record, aParent, aIndex);
                });
            break;
        default:
            break;
        }
    },

    processCouchBookmarkRevision: function(record, aParent, aIndex) {
        // Could be an add or change revision. Delete was handled earlier.
        // If it's an addition (we can't resolve its _id to be one of our itemIds),
        // add it to the Desktop Couch folder in unfiled.
        Bindwood.writeMessage("Processing bookmark record: " + JSON.stringify(record));
        var itemId = Bindwood.itemIdForUUID(record._id);
        if (itemId) {
            // It's a change. Stamp everything remote on the local bookmark
            bookmarksService.setItemTitle(itemId, record.title);
            bookmarksService.changeBookmarkURI(itemId,
                ioService.newURI(record.uri, null, null));
        } else {
            // It's an addition. Add a new bookmark to our scratch folder,
            // annotate it, and we're done.
            itemId = bookmarksService.insertBookmark(
                aParent,
                ioService.newURI(record.uri, null, null),
                aIndex,
                record.title);
            Bindwood.annotateItemWithUUID(itemId, record._id);
        }
    },

    processCouchFolderRevision: function(record, aParent, aIndex) {
        // Could be an add or change revision. Delete was handled earlier.
        // If it's an addition (we can't resolve its _id to be one of our itemIds),
        // add it to the Desktop Couch folder in unfiled.
        Bindwood.writeMessage("Processing folder record: " + JSON.stringify(record));
        var itemId = Bindwood.itemIdForUUID(record._id);
        if (itemId) {
            // It's a change. Stamp remote title on the folder, and deal with any
            // changed children.
            Bindwood.noteStartTime('Shuffling folder children');
            bookmarksService.setItemTitle(itemId, record.title);
            // Iterate through our current folder children, and compare with remote.
            // Move all local children to the scratch folder, then move them back
            // in the order of the remote children.
            var local_children = Bindwood.getUUIDsFromFolder(itemId);
            Bindwood.writeMessage("Moving local children " + JSON.stringify(local_children) + " to scratch folder");
            for (var i = 0; i<local_children.length; i++) {
                var child = local_children[i];
                var child_itemId = Bindwood.itemIdForUUID(child);
                try {
                    bookmarksService.moveItem(child_itemId, Bindwood.scratch_folder, -1);
                } catch(e) {
                    Bindwood.writeError("Problem moving item to scratch folder: " + JSON.stringify(e), e);
                }
            }
            Bindwood.writeMessage("Moving children identified by record " + JSON.stringify(record.children) + " to this folder");
            for (var j = 0; j<record.children.length; j++) {
                var new_child = record.children[j];
                var new_child_itemId = Bindwood.itemIdForUUID(new_child);
                try {
                    bookmarksService.moveItem(new_child_itemId, itemId, -1);
                } catch(e) {
                    Bindwood.writeError("Problem moving item from scratch folder: " + JSON.stringify(e), e);
                }
            }
            Bindwood.noteEndTime('Shuffling folder children');
        } else {
            // It's an addition. Add a new bookmark to our scratch folder,
            // annotate it, and we're done.
            itemId = bookmarksService.createFolder(
                aParent,
                record.title,
                aIndex);
            Bindwood.annotateItemWithUUID(itemId, record._id);
        }
    },

    processCouchFeedRevision: function(record, aParent, aIndex) {
        // Could be an add or change revision. Delete was handled earlier.
        // If it's an addition (we can't resolve its _id to be one of our itemIds),
        // add it to the Desktop Couch folder in unfiled.
        Bindwood.writeMessage("Processing feed record: " + JSON.stringify(record));
        var itemId = Bindwood.itemIdForUUID(record._id);
        if (itemId) {
            // It's a change. Stamp everything remote on the local bookmark
            bookmarksService.setItemTitle(itemId, record.title);
            livemarkService.setSiteURI(itemId,
                ioService.newURI(record.site_uri, null, null));
            livemarkService.setFeedURI(itemId,
                ioService.newURI(record.feed_uri, null, null));
        } else {
            // It's an addition. Add a new bookmark to our scratch folder,
            // annotate it, and we're done.
            var newItemId = livemarkService.createLivemarkFolderOnly(
                bookmarksService,
                aParent,
                record.title,
                ioService.newURI(record.site_uri, null, null),
                ioService.newURI(record.feed_uri, null, null),
                aIndex);
            Bindwood.annotateItemWithUUID(newItemId, record._id);
        }
    },

    processCouchSeparatorRevision: function(record, aParent, aIndex) {
        // Should only be an add revision. There's nothing to change, and delete was
        // handled earlier.
        // If it's an addition (we can't resolve its _id to be one of our itemIds),
        // add it to the Desktop Couch folder in unfiled.
        Bindwood.writeMessage("Processing separator record: " + JSON.stringify(record));
        var itemId = Bindwood.itemIdForUUID(record._id);
        if (!itemId) {
            // There's nothing to change about a separator, so...
            // It's an addition. Add a new bookmark to our scratch folder,
            // annotate it, and we're done.
            var newItemId = bookmarksService.insertSeparator(
                aParent,
                aIndex);
            Bindwood.annotateItemWithUUID(newItemId, record._id);
        }
    },

    updateDocAndSave: function(uuid, attribute, value, callback) {
        Bindwood.writeMessage(
            "Updating a document (" +
                uuid +
                ") setting (" +
                attribute +
                ") to (" + value + ")");

        // Some attributes that we track should remain inside the
        // application_annotations object
        var attrMap = {
             title: true,
             uri: true,
             feed_uri: true,
             site_uri: true,
             children: true,
             favicon: false,
             profile: false };

        var doc = Bindwood.couch.open(uuid);
        if (attrMap[attribute.toString()] || false) { // belongs at top-level
            doc[attribute.toString()] = value.toString();
        } else {
            if (!doc.application_annotations) {
                doc.application_annotations = {};
            }
            if (!doc.application_annotations.Firefox) {
                doc.application_annotations.Firefox = {};
            }
            doc.application_annotations.Firefox[attribute.toString()] = value.toString();
        }
        try {
            var response = Bindwood.couch.save(doc);
            Bindwood.seen_revisions[response.rev] = true;
        } catch(e) {
            Bindwood.writeError("Problem saving document to Couch", e);
            throw e;
        }

        if (callback) {
            callback();
        }

        return response;
    },

    itemWeCareAbout: function(itemId) {
        // Traverse from the itemId up its parent chain. If at any
        // level the parent is a livemark container or a dynamic
        // container, return false, otherwise, return true.
        var root = 0;
        var parent;
        while (parent != root) {
            Bindwood.writeMessage("Looking for parent of " + itemId);
            parent = bookmarksService.getFolderIdForItem(itemId);
            if (parent != root &&
                annotationService.itemHasAnnotation(
                    parent, 'livemark/feedURI')) {
                return false;
            }
            itemId = parent;
        }
        return true;
    },

    Observer: {
        // An nsINavBookmarkObserver
        onItemAdded: function(aItemId, aFolder, aIndex) {
            Bindwood.writeMessage("onItemAdded: called when push is " + Bindwood.push);
            // An item has been added, so we create a blank entry
            // in Couch with our local itemId attached.
            if (!Bindwood.itemWeCareAbout(aItemId)) {
                Bindwood.writeMessage("Ignoring this add event");
                return;
            }

            Bindwood.writeMessage(
                "A new item was created. Its id is: " + aItemId +
                    " at location: " + aIndex +
                    " in folder: " + aFolder );
            netscape.security.PrivilegeManager.enablePrivilege(
                "UniversalBrowserRead UniversalBrowserWrite");

            switch (Bindwood.push) {
            case 'DISABLED':
                Bindwood.writeMessage("Added, but not saving to Couch.");
                break;
            case 'ENABLED':
                try {
                    var doc = Bindwood.couchRecordForItemId(aItemId);
                    var response = Bindwood.couch.save(doc);
                    Bindwood.writeMessage("Saved new, bare record to Couch.");
                    Bindwood.seen_revisions[response.rev] = true;
                    Bindwood.pushFolderChildren(aFolder);
                } catch(e) {
                    Bindwood.writeError(
                        "Problem saving new bookmark to Couch: ", e);
                }
                break;
            default:
                break;
            }

            Bindwood.setLatestModified(
                bookmarksService.getItemLastModified(aItemId));
        },
        onBeforeItemRemoved: function(aItemId) {
            Bindwood.writeMessage("onBeforeItemRemoved: called when push is " + Bindwood.push);
            // A bookmark has been removed. This is called before it's
            // been removed locally, though we're passed the itemId,
            // which we use to delete from Couch.
            var folderId = bookmarksService.getFolderIdForItem(aItemId);
            if (!Bindwood.itemWeCareAbout(aItemId)) {
                Bindwood.writeMessage("Ignoring this before remove event");
                return;
            }

            Bindwood.writeMessage(
                "Record " + aItemId + " is about to be removed locally.");
            var uuid = Bindwood.uuidForItemId(aItemId);

            switch (Bindwood.push) {
            case 'DISABLED':
                delete Bindwood.uuidItemIdMap[uuid];
                Bindwood.writeMessage(
                    "Deleted from local uuid map, but not saving back to Couch.");
                break;
            case 'ENABLED':
                netscape.security.PrivilegeManager.enablePrivilege(
                    "UniversalBrowserRead UniversalBrowserWrite");

                var doc = Bindwood.couch.open(uuid);
                if (!doc.application_annotations) {
                    doc.application_annotations = {};
                }
                if (!doc.application_annotations['Ubuntu One']) {
                    doc.application_annotations['Ubuntu One'] = {};
                }
                if (!doc.application_annotations['Ubuntu One'].private_application_annotations) {
                    doc.application_annotations['Ubuntu One'].private_application_annotations = {};
                }
                doc.application_annotations['Ubuntu One'].private_application_annotations.deleted = true;
                
                try {
                    // Also remove from our local cache and remove
                    // annotation from service.
                    var response = Bindwood.couch.save(doc);
                    Bindwood.seen_revisions[response.rev] = true;
                    delete Bindwood.uuidItemIdMap[uuid];
                    Bindwood.writeMessage(
                        "Deleted local reference in the" +
                            " uuid-itemId mapping.");
                    Bindwood.writeMessage(
                        "Saved document back to Couch with deleted flag set.");
                    var new_children = Bindwood.getUUIDsFromFolder(folderId);
                    new_children.splice(new_children.indexOf(uuid), 1);
                    Bindwood.pushFolderChildren(folderId, new_children);
                } catch(e) {
                    Bindwood.writeError(
                        "Problem pushing deleted record to Couch: ", e);
                }
                break;
            default:
                break;
            }
        },
        onItemRemoved: function(aItemId, aFolder, aIndex) {
            Bindwood.writeMessage("onItemRemoved: called when push is " + Bindwood.push);
            // This only happens locally, so there's never a need to push
            if (!Bindwood.itemWeCareAbout(aItemId)) {
                Bindwood.writeMessage("Ignoring this remove event");
                return;
            }

            Bindwood.makeLocalChangeOnly(
                function() {
                    return annotationService.removeItemAnnotation(
                        aItemId, Bindwood.annotationKey); });
            Bindwood.writeMessage(
                "Removed annotations from bookmark identified by: " + aItemId);
        },
        onItemChanged: function(aItemId, aProperty, aIsAnnotationProperty, aValue) {
            Bindwood.writeMessage("onItemChanged: called when push is " + Bindwood.push);
            // A property of a bookmark has changed. On multiple
            // property updates, this will be called multiple times,
            // once per property (i.e., for title and URI)
            if (!Bindwood.itemWeCareAbout(aItemId)) {
                Bindwood.writeMessage("Ignoring this change event");
                return;
            }

            Bindwood.writeMessage(
                "A property (" +
                    aProperty +
                    ") on item id: " + aItemId +
                    " has been set to: " + aValue);
            var uuid = Bindwood.uuidForItemId(aItemId);

            switch (Bindwood.push) {
            case 'DISABLED':
                Bindwood.writeMessage(
                    "Updated, but not saving back to Couch.");
                break;
            case 'ENABLED':
                Bindwood.writeMessage(
                    "We will push this change back to Couch.");
                netscape.security.PrivilegeManager.enablePrivilege(
                    "UniversalBrowserRead UniversalBrowserWrite");
                try {
                    var result = Bindwood.updateDocAndSave(
                        uuid, aProperty.toString(), aValue.toString(),
                        function() {
                            Bindwood.writeMessage(
                                "Saved the document back to Couch"); });
                } catch(e) {
                    Bindwood.writeError(
                        "Problem saving updated bookmark to Couch: ", e);
                }
                break;
            default:
                break;
            }

            Bindwood.setLatestModified(
                bookmarksService.getItemLastModified(aItemId));
        },

        onItemMoved: function(aItemId, aOldParent, aOldIndex, aNewParent, aNewIndex) {
            Bindwood.writeMessage("onItemMoved: called when push is " + Bindwood.push);
            Bindwood.writeMessage(
                "The item: " + aItemId + " was moved from (" + aOldParent + ", " + aOldIndex +
                ") to (" + aNewParent + ", " + aNewIndex + ")"
            );
            switch (Bindwood.push) {
            case 'DISABLED':
                Bindwood.writeMessage(
                    "Moved, but not saving back to Couch.");
                break;
            case 'ENABLED':
                var uuid = Bindwood.uuidForItemId(aItemId);
                var old_parent_uuid = Bindwood.uuidForItemId(aOldParent);
                var old_parent_doc = Bindwood.couch.open(old_parent_uuid);
                old_parent_doc.children = Bindwood.getUUIDsFromFolder(aOldParent);
                try {
                    var response = Bindwood.couch.save(old_parent_doc);
                    Bindwood.seen_revisions[response.rev] = true;
                } catch(e) {
                    Bindwood.writeError(
                        "Problem saving updated old parent doc to Couch: ", e);
                }
                if (aOldParent != aNewParent) {
                    var new_parent_uuid = Bindwood.uuidForItemId(aNewParent);
                    var new_parent_doc = Bindwood.couch.open(new_parent_uuid);
                    new_parent_doc.children = Bindwood.getUUIDsFromFolder(aNewParent);
                    try {
                        var response = Bindwood.couch.save(new_parent_doc);
                        Bindwood.seen_revisions[response.rev] = true;
                    } catch(e) {
                        Bindwood.writeError(
                            "Problem saving updated new parent doc to Couch: ", e);
                    }
                }
                break;
            default:
                break;
            }

            // Set the latest modified to the greatest of aItemId, aOldParent, or aNewParent's last_modified
            Bindwood.setLatestModified(
                [bookmarksService.getItemLastModified(aItemId),
                 bookmarksService.getItemLastModified(aOldParent),
                 bookmarksService.getItemLastModified(aNewParent)].sort()[2]);
        },

        // Currently unhandled
        onBeginUpdateBatch: function() {},
        onEndUpdateBatch: function() {},
        onItemVisited: function(aBookmarkId, aVisitID, time) {},

        // Query Interface
        QueryInterface: function(iid) {
            if (iid.equals(Ci.nsINavBookmarkObserver) ||
                iid.equals(Ci.nsINavBookmarkObserver_MOZILLA_1_9_1_ADDITIONS) ||
                iid.equals(Ci.nsISupports)) {
                return this;
            }
            throw Cr.NS_ERROR_NO_INTERFACE;
        }
    }
};