← Back to branch summary

~duplicity-team/duplicity/0.6-series

« back to all changes in this revision

Viewing changes to duplicity/backends/gdocsbackend.py

  • Committer: Kenneth Loafman
  • Date: 2011-08-06 15:57:54 UTC
  • mfrom: (761.4.15 bzr)
  • Revision ID: kenneth@loafman.com-20110806155754-up25y4q3dy9zdphw
Merged in lp:~carlos-abalde/duplicity/google-docs

Show diffs side-by-side

added added

removed removed

Lines of Context:
duplicity/backends/gdocsbackend.py
 
1
# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 
2
#
 
3
# Copyright 2011 Carlos Abalde <carlos.abalde@gmail.com>
 
4
#
 
5
# This file is part of duplicity.
 
6
#
 
7
# Duplicity is free software; you can redistribute it and/or modify it
 
8
# under the terms of the GNU General Public License as published by the
 
9
# Free Software Foundation; either version 2 of the License, or (at your
 
10
# option) any later version.
 
11
#
 
12
# Duplicity is distributed in the hope that it will be useful, but
 
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
 
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 
15
# General Public License for more details.
 
16
#
 
17
# You should have received a copy of the GNU General Public License
 
18
# along with duplicity; if not, write to the Free Software Foundation,
 
19
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
20
 
 
21
import os.path
 
22
import string
 
23
import urllib;
 
24
 
 
25
import duplicity.backend
 
26
from duplicity.backend import retry
 
27
from duplicity import log
 
28
from duplicity.errors import * #@UnusedWildImport
 
29
 
 
30
class GDocsBackend(duplicity.backend.Backend):
 
31
    """Connect to remote store using Google Google Documents List API"""
 
32
 
 
33
    ROOT_FOLDER_ID = 'folder%3Aroot'
 
34
    BACKUP_DOCUMENT_TYPE = 'application/binary'
 
35
 
 
36
    def __init__(self, parsed_url):
 
37
        duplicity.backend.Backend.__init__(self, parsed_url)
 
38
 
 
39
        # Import Google Data APIs libraries.
 
40
        try:
 
41
            global atom
 
42
            global gdata
 
43
            import atom.data
 
44
            import gdata.client
 
45
            import gdata.docs.client
 
46
            import gdata.docs.data
 
47
        except ImportError:
 
48
            raise BackendException('Google Docs backend requires Google Data APIs Python '
 
49
                                   'Client Library (see http://code.google.com/p/gdata-python-client/).')
 
50
 
 
51
        # Setup client instance.
 
52
        self.client = gdata.docs.client.DocsClient(source='duplicity $version')
 
53
        self.client.ssl = True
 
54
        self.client.http_client.debug = False
 
55
        self.__authorize(parsed_url.username + '@' + parsed_url.hostname, self.get_password())
 
56
 
 
57
        # Fetch destination folder entry (and crete hierarchy if required).
 
58
        folder_names = string.split(parsed_url.path[1:], '/')
 
59
        parent_folder = None
 
60
        parent_folder_id = GDocsBackend.ROOT_FOLDER_ID
 
61
        for folder_name in folder_names:
 
62
            entries = self.__fetch_entries(parent_folder_id, 'folder', folder_name)
 
63
            if entries is not None:
 
64
                if len(entries) == 1:
 
65
                    parent_folder = entries[0]
 
66
                elif len(entries) == 0:
 
67
                    parent_folder = self.client.create(gdata.docs.data.FOLDER_LABEL, folder_name, parent_folder)
 
68
                else:
 
69
                    parent_folder = None
 
70
                if parent_folder:
 
71
                    parent_folder_id = parent_folder.resource_id.text
 
72
                else:
 
73
                    raise BackendException("Error while creating destination folder '%s'." % folder_name)
 
74
            else:
 
75
                raise BackendException("Error while fetching destination folder '%s'." % folder_name)
 
76
        self.folder = parent_folder
 
77
 
 
78
    @retry
 
79
    def put(self, source_path, remote_filename=None, raise_errors = False):
 
80
        """Transfer source_path to remote_filename"""
 
81
        # Default remote file name.
 
82
        if not remote_filename:
 
83
            remote_filename = source_path.get_filename()
 
84
 
 
85
        # Upload!
 
86
        try:
 
87
            # If remote file already exists in destination folder, remove it.
 
88
            entries = self.__fetch_entries(self.folder.resource_id.text,
 
89
                                           GDocsBackend.BACKUP_DOCUMENT_TYPE,
 
90
                                           remote_filename)
 
91
            for entry in entries:
 
92
                self.client.delete(entry.get_edit_link().href + '?delete=true', force=True)
 
93
            
 
94
            # Set uploader instance. Note that resumable uploads are required in order to
 
95
            # enable uploads for all file types.
 
96
            # (see http://googleappsdeveloper.blogspot.com/2011/05/upload-all-file-types-to-any-google.html)
 
97
            file = source_path.open()
 
98
            uploader = gdata.client.ResumableUploader(
 
99
              self.client, file, GDocsBackend.BACKUP_DOCUMENT_TYPE, os.path.getsize(file.name),
 
100
              chunk_size=gdata.client.ResumableUploader.DEFAULT_CHUNK_SIZE,
 
101
              desired_class=gdata.docs.data.DocsEntry)
 
102
            if uploader:
 
103
                # Chunked upload.
 
104
                entry = gdata.docs.data.DocsEntry(title = atom.data.Title(text = remote_filename))
 
105
                uri = '/feeds/upload/create-session/default/private/full?convert=false'
 
106
                entry = uploader.UploadFile(uri, entry = entry)
 
107
                if entry:
 
108
                    # Move to destination folder.
 
109
                    # TODO: any ideas on how to avoid this step?
 
110
                    if self.client.Move(entry, self.folder):
 
111
                        assert not file.close()
 
112
                        return
 
113
                    else:
 
114
                        self.__handle_error("Failed to move uploaded file '%s' to destination remote folder '%s'"
 
115
                                            % (source_path.get_filename(), self.folder.title.text), raise_errors)
 
116
                else:
 
117
                    self.__handle_error("Failed to upload file '%s' to remote folder '%s'" 
 
118
                                        % (source_path.get_filename(), self.folder.title.text), raise_errors)
 
119
            else:
 
120
                self.__handle_error("Failed to initialize upload of file '%s' to remote folder '%s'"
 
121
                         % (source_path.get_filename(), self.folder.title.text), raise_errors)
 
122
            assert not file.close()
 
123
        except Exception, e:
 
124
            self.__handle_error("Failed to upload file '%s' to remote folder '%s': %s"
 
125
                                % (source_path.get_filename(), self.folder.title.text, str(e)), raise_errors)
 
126
 
 
127
    @retry
 
128
    def get(self, remote_filename, local_path, raise_errors = False):
 
129
        """Get remote filename, saving it to local_path"""
 
130
        try:
 
131
            entries = self.__fetch_entries(self.folder.resource_id.text,
 
132
                                           GDocsBackend.BACKUP_DOCUMENT_TYPE,
 
133
                                           remote_filename)
 
134
            if len(entries) == 1:
 
135
                entry = entries[0]
 
136
                self.client.Download(entry, local_path.name)
 
137
                local_path.setdata()
 
138
                return
 
139
            else:
 
140
                self.__handle_error("Failed to find file '%s' in remote folder '%s'"
 
141
                                    % (remote_filename, self.folder.title.text), raise_errors)
 
142
        except Exception, e:
 
143
            self.__handle_error("Failed to download file '%s' in remote folder '%s': %s"
 
144
                                 % (remote_filename, self.folder.title.text, str(e)), raise_errors)
 
145
 
 
146
    @retry
 
147
    def list(self, raise_errors = False):
 
148
        """List files in folder"""
 
149
        try:
 
150
            entries = self.__fetch_entries(self.folder.resource_id.text,
 
151
                                           GDocsBackend.BACKUP_DOCUMENT_TYPE)
 
152
            return [entry.title.text for entry in entries]
 
153
        except Exception, e:
 
154
            self.__handle_error("Failed to fetch list of files in remote folder '%s': %s"
 
155
                                % (self.folder.title.text, str(e)), raise_errors)
 
156
 
 
157
    @retry
 
158
    def delete(self, filename_list, raise_errors = False):
 
159
        """Delete files in filename_list"""
 
160
        for filename in filename_list:
 
161
            try:
 
162
                entries = self.__fetch_entries(self.folder.resource_id.text,
 
163
                                               GDocsBackend.BACKUP_DOCUMENT_TYPE,
 
164
                                               filename)
 
165
                if len(entries) > 0:
 
166
                    success = True
 
167
                    for entry in entries:
 
168
                        if not self.client.delete(entry.get_edit_link().href + '?delete=true', force = True):
 
169
                            success = False
 
170
                    if not success:
 
171
                        self.__handle_error("Failed to remove file '%s' in remote folder '%s'"
 
172
                                            % (filename, self.folder.title.text), raise_errors)
 
173
                else:
 
174
                    log.Warn("Failed to fetch file '%s' in remote folder '%s'"
 
175
                             % (filename, self.folder.title.text))
 
176
            except Exception, e:
 
177
                self.__handle_error("Failed to remove file '%s' in remote folder '%s': %s"
 
178
                                    % (filename, self.folder.title.text, str(e)), raise_errors)
 
179
 
 
180
    def __handle_error(self, message, raise_errors = True):
 
181
        if raise_errors:
 
182
            raise BackendException(message)
 
183
        else:
 
184
            log.FatalError(message, log.ErrorCode.backend_error)
 
185
    
 
186
    def __authorize(self, email, password, captcha_token = None, captcha_response = None):
 
187
        try:
 
188
            self.client.client_login(email,
 
189
                                     password,
 
190
                                     source = 'duplicity $version',
 
191
                                     service = 'writely',
 
192
                                     captcha_token = captcha_token,
 
193
                                     captcha_response = captcha_response)
 
194
        except gdata.client.CaptchaChallenge, challenge:
 
195
            print('A captcha challenge in required. Please visit ' + challenge.captcha_url)
 
196
            answer = None
 
197
            while not answer:
 
198
                answer = raw_input('Answer to the challenge? ')
 
199
            self.__authorize(email, password, challenge.captcha_token, answer)
 
200
        except gdata.client.BadAuthentication:
 
201
            self.__handle_error('Invalid user credentials given. Be aware that accounts '
 
202
                                'that use 2-step verification require creating an application specific '
 
203
                                'access code for using this Duplicity backend. Follow the instrucction in '
 
204
                                'http://www.google.com/support/accounts/bin/static.py?page=guide.cs&guide=1056283&topic=1056286 '
 
205
                                'and create your application-specific password to run duplicity backups.')
 
206
        except Exception, e:
 
207
            self.__handle_error('Error while authenticating client: %s.' % str(e))
 
208
 
 
209
    def __fetch_entries(self, folder_id, type, title = None):
 
210
        # Build URI.
 
211
        uri = '/feeds/default/private/full/%s/contents' % folder_id
 
212
        if type == 'folder':
 
213
            uri += '/-/folder?showfolders=true'
 
214
        elif type == GDocsBackend.BACKUP_DOCUMENT_TYPE:
 
215
            uri += '?showfolders=false'
 
216
        else:
 
217
            uri += '?showfolders=true'
 
218
        if title:
 
219
            uri += '&title=' + urllib.quote(title) + '&title-exact=true'
 
220
        
 
221
        try:
 
222
            # Fetch entries
 
223
            entries = self.client.get_everything(uri = uri)
 
224
            
 
225
            # When filtering by entry title, API is returning (don't know why) documents in other
 
226
            # folders (apart from folder_id) matching the title, so some extra filtering is required.
 
227
            if title:
 
228
                result = []
 
229
                for entry in entries:
 
230
                    if (not type) or (entry.get_document_type() == type):
 
231
                        if folder_id != GDocsBackend.ROOT_FOLDER_ID:
 
232
                            for link in entry.in_folders():
 
233
                                folder_entry = self.client.get_entry(link.href, None, None,
 
234
                                                                     desired_class=gdata.docs.data.DocsEntry)
 
235
                                if folder_entry and (folder_entry.resource_id.text == folder_id):
 
236
                                    result.append(entry)
 
237
                        elif len(entry.in_folders()) == 0:
 
238
                            result.append(entry)
 
239
            else:
 
240
                result = entries
 
241
            
 
242
            # Done!
 
243
            return result
 
244
        except Exception, e:
 
245
            self.__handle_error('Error while fetching remote entries: %s.' % str(e))
 
246
 
 
247
duplicity.backend.register_backend('gdocs', GDocsBackend)