← Back to branch summary

~jaroslaw-rosiek/duplicity/expose-backends

« back to all changes in this revision

Viewing changes to duplicity/backends/pydrivebackend.py

  • Committer: Kenneth Loafman
  • Date: 2015-09-14 20:40:21 UTC
  • mfrom: (1100.1.5 pydrive-id-cache)
  • Revision ID: kenneth@loafman.com-20150914204021-uzmba1c0srii0u4e
* Merged in lp:~bmerry/duplicity/pydrive-id-cache
  - This fixes the issue a number of users (including myself) have been
    having with duplicity creating files with duplicate filenames on
    Google Drive. It keeps a runtime cache of filename to object ID
    mappings, so that once it has uploaded an object it won't be fooled
    by weakly consistent directory listings.

Show diffs side-by-side

added added

removed removed

Lines of Context:
duplicity/backends/pydrivebackend.py
20
20
import os
21
21
 
22
22
import duplicity.backend
 
23
from duplicity import log
23
24
from duplicity.errors import BackendException
24
25
 
25
26
 
35
36
            from oauth2client.client import SignedJwtAssertionCredentials
36
37
            from pydrive.auth import GoogleAuth
37
38
            from pydrive.drive import GoogleDrive
 
39
            from pydrive.files import FileNotUploadedError
38
40
        except ImportError:
39
41
            raise BackendException('PyDrive backend requires PyDrive installation'
40
42
                                   'Please read the manpage to fix.')
73
75
                folder.Upload()
74
76
            parent_folder_id = folder['id']
75
77
        self.folder = parent_folder_id
76
 
 
77
 
    def FilesList(self):
78
 
        return self.drive.ListFile({'q': "'" + self.folder + "' in parents and trashed=false"}).GetList()
 
78
        self.id_cache = {}
 
79
 
 
80
    def file_by_name(self, filename):
 
81
        from pydrive.files import ApiRequestError
 
82
        if filename in self.id_cache:
 
83
            # It might since have been locally moved, renamed or deleted, so we
 
84
            # need to validate the entry.
 
85
            file_id = self.id_cache[filename]
 
86
            drive_file = self.drive.CreateFile({'id': file_id})
 
87
            try:
 
88
                if drive_file['title'] == filename and not drive_file['labels']['trashed']:
 
89
                    for parent in drive_file['parents']:
 
90
                        if parent['id'] == self.folder:
 
91
                            log.Info("PyDrive backend: found file '%s' with id %s in ID cache" % (filename, file_id))
 
92
                            return drive_file
 
93
            except ApiRequestError as error:
 
94
                # A 404 occurs if the ID is no longer valid
 
95
                if error.args[0].resp.status != 404:
 
96
                    raise
 
97
            # If we get here, the cache entry is invalid
 
98
            log.Info("PyDrive backend: invalidating '%s' (previously ID %s) from ID cache" % (filename, file_id))
 
99
            del self.id_cache[filename]
 
100
 
 
101
        # Not found in the cache, so use directory listing. This is less
 
102
        # reliable because there is no strong consistency.
 
103
        q = "title='%s' and '%s' in parents and trashed=false" % (filename, self.folder)
 
104
        fields = 'items(title,id,fileSize,downloadUrl,exportLinks),nextPageToken'
 
105
        flist = self.drive.ListFile({'q': q, 'fields': fields}).GetList()
 
106
        if len(flist) > 1:
 
107
            log.FatalError(_("PyDrive backend: multiple files called '%s'.") % (filename,))
 
108
        elif flist:
 
109
            file_id = flist[0]['id']
 
110
            self.id_cache[filename] = flist[0]['id']
 
111
            log.Info("PyDrive backend: found file '%s' with id %s on server, adding to cache" % (filename, file_id))
 
112
            return flist[0]
 
113
        log.Info("PyDrive backend: file '%s' not found in cache or on server" % (filename,))
 
114
        return None
79
115
 
80
116
    def id_by_name(self, filename):
81
 
        try:
82
 
            return next(item for item in self.FilesList() if item['title'] == filename)['id']
83
 
        except:
 
117
        drive_file = self.file_by_name(filename)
 
118
        if drive_file is None:
84
119
            return ''
 
120
        else:
 
121
            return drive_file['id']
85
122
 
86
123
    def _put(self, source_path, remote_filename):
87
 
        drive_file = self.drive.CreateFile({'title': remote_filename, 'parents': [{"kind": "drive#fileLink", "id": self.folder}]})
 
124
        drive_file = self.file_by_name(remote_filename)
 
125
        if drive_file is None:
 
126
            # No existing file, make a new one
 
127
            drive_file = self.drive.CreateFile({'title': remote_filename, 'parents': [{"kind": "drive#fileLink", "id": self.folder}]})
 
128
            log.Info("PyDrive backend: creating new file '%s'" % (remote_filename,))
 
129
        else:
 
130
            log.Info("PyDrive backend: replacing existing file '%s' with id '%s'" % (
 
131
                remote_filename, drive_file['id']))
88
132
        drive_file.SetContentFile(source_path.name)
89
133
        drive_file.Upload()
 
134
        self.id_cache[remote_filename] = drive_file['id']
90
135
 
91
136
    def _get(self, remote_filename, local_path):
92
 
        drive_file = self.drive.CreateFile({'id': self.id_by_name(remote_filename)})
 
137
        drive_file = self.file_by_name(remote_filename)
93
138
        drive_file.GetContentFile(local_path.name)
94
139
 
95
140
    def _list(self):
96
 
        return [item['title'] for item in self.FilesList()]
 
141
        drive_files = self.drive.ListFile({
 
142
            'q': "'" + self.folder + "' in parents and trashed=false",
 
143
            'fields': 'items(title,id),nextPageToken'}).GetList()
 
144
        filenames = set(item['title'] for item in drive_files)
 
145
        # Check the cache as well. A file might have just been uploaded but
 
146
        # not yet appear in the listing.
 
147
        # Note: do not use iterkeys() here, because file_by_name will modify
 
148
        # the cache if it finds invalid entries.
 
149
        for filename in self.id_cache.keys():
 
150
            if (filename not in filenames) and (self.file_by_name(filename) is not None):
 
151
                filenames.add(filename)
 
152
        return list(filenames)
97
153
 
98
154
    def _delete(self, filename):
99
155
        file_id = self.id_by_name(filename)
100
 
        drive_file = self.drive.CreateFile({'id': file_id})
101
 
        drive_file.auth.service.files().delete(fileId=drive_file['id']).execute()
102
 
 
103
 
    def _delete_list(self, filename_list):
104
 
        to_remove = set(filename_list)
105
 
        for item in self.FilesList():
106
 
            if item['title'] not in to_remove:
107
 
                continue
108
 
            file_id = item['id']
109
 
            drive_file = self.drive.CreateFile({'id': file_id})
110
 
            drive_file.auth.service.files().delete(fileId=drive_file['id']).execute()
 
156
        if file_id != '':
 
157
            self.drive.auth.service.files().delete(fileId=file_id).execute()
 
158
        else:
 
159
            log.Warn("File '%s' does not exist while trying to delete it" % (filename,))
111
160
 
112
161
    def _query(self, filename):
113
 
        try:
114
 
            size = int((item for item in self.FilesList() if item['title'] == filename).next()['fileSize'])
115
 
        except:
 
162
        drive_file = self.file_by_name(filename)
 
163
        if drive_file is None:
116
164
            size = -1
 
165
        else:
 
166
            size = int(drive_file['fileSize'])
117
167
        return {'size': size}
118
168
 
 
169
    def _error_code(self, operation, error):
 
170
        from pydrive.files import ApiRequestError, FileNotUploadedError
 
171
        if isinstance(error, FileNotUploadedError):
 
172
            return log.ErrorCode.backend_not_found
 
173
        elif isinstance(error, ApiRequestError):
 
174
            http_status = error.args[0].resp.status
 
175
            if http_status == 404:
 
176
                return log.ErrorCode.backend_not_found
 
177
            elif http_status == 403:
 
178
                return log.ErrorCode.backend_permission_denied
 
179
        return log.ErrorCode.backend_error
 
180
 
119
181
duplicity.backend.register_backend('pydrive', PyDriveBackend)
120
182
""" pydrive is an alternate way to access gdocs """
121
183
duplicity.backend.register_backend('pydrive+gdocs', PyDriveBackend)