« back to all changes in this revision
Viewing changes to duplicity/backends/_cf_pyrax.py
- browse files at revision 978.2.1
- compare with another revision
- download diff
- download tarball
- view history from revision 978.2.1
- Committer: Michael Terry
- Date: 2014-04-21 19:21:45 UTC
- mto: This revision was merged to the branch mainline in revision 981.
- Revision ID: michael.terry@canonical.com-20140421192145-b1vlb0hppnn8jrtl
Checkpoint
- files added:
- duplicity/backends/README
- files modified:
- bin/duplicity
added
removed
duplicity/backends/_cf_pyrax.py
19
19
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20
20
21
21
import os
22
import time
23
22
24
23
import duplicity.backend
25
from duplicity import globals
26
24
from duplicity import log
27
from duplicity.errors import * # @UnusedWildImport
28
from duplicity.util import exception_traceback
29
from duplicity.backend import retry
25
from duplicity.errors import BackendException
26
30
27
31
28
class PyraxBackend(duplicity.backend.Backend):
32
29
"""
72
69
self.cloudfiles = pyrax.cloudfiles
73
70
self.container = pyrax.cloudfiles.create_container(container)
74
71
75
def put(self, source_path, remote_filename = None):
76
if not remote_filename:
77
remote_filename = source_path.get_filename()
78
79
for n in range(1, globals.num_retries + 1):
80
log.Info("Uploading '%s/%s' " % (self.container, remote_filename))
81
try:
82
self.container.upload_file(source_path.name, remote_filename)
83
return
84
except self.client_exc as error:
85
log.Warn("Upload of '%s' failed (attempt %d): pyrax returned: %s %s"
86
% (remote_filename, n, error.__class__.__name__, error.message))
87
except Exception as e:
88
log.Warn("Upload of '%s' failed (attempt %s): %s: %s"
89
% (remote_filename, n, e.__class__.__name__, str(e)))
90
log.Debug("Backtrace of previous error: %s"
91
% exception_traceback())
92
time.sleep(30)
93
log.Warn("Giving up uploading '%s' after %s attempts"
94
% (remote_filename, globals.num_retries))
95
raise BackendException("Error uploading '%s'" % remote_filename)
96
97
def get(self, remote_filename, local_path):
98
for n in range(1, globals.num_retries + 1):
99
log.Info("Downloading '%s/%s'" % (self.container, remote_filename))
100
try:
101
sobject = self.container.get_object(remote_filename)
102
f = open(local_path.name, 'w')
72
def _error_code(self, e):
73
if isinstance(e, self.client_exc):
74
if e.status == 404:
75
return log.ErrorCode.backend_not_found
76
77
def _put(self, source_path, remote_filename):
78
self.container.upload_file(source_path.name, remote_filename)
79
80
def _get(self, remote_filename, local_path):
81
try:
82
sobject = self.container.get_object(remote_filename)
83
with open(local_path.name, 'wb') as f:
103
84
f.write(sobject.get())
104
local_path.setdata()
105
return
106
except self.nso_exc:
107
return
108
except self.client_exc as resperr:
109
log.Warn("Download of '%s' failed (attempt %s): pyrax returned: %s %s"
110
% (remote_filename, n, resperr.__class__.__name__, resperr.message))
111
except Exception as e:
112
log.Warn("Download of '%s' failed (attempt %s): %s: %s"
113
% (remote_filename, n, e.__class__.__name__, str(e)))
114
log.Debug("Backtrace of previous error: %s"
115
% exception_traceback())
116
time.sleep(30)
117
log.Warn("Giving up downloading '%s' after %s attempts"
118
% (remote_filename, globals.num_retries))
119
raise BackendException("Error downloading '%s/%s'"
120
% (self.container, remote_filename))
85
except self.nso_exc:
86
pass
121
87
122
88
def _list(self):
123
for n in range(1, globals.num_retries + 1):
124
log.Info("Listing '%s'" % (self.container))
125
try:
126
# Cloud Files will return a max of 10,000 objects. We have
127
# to make multiple requests to get them all.
128
objs = self.container.get_object_names()
129
keys = objs
130
while len(objs) == 10000:
131
objs = self.container.get_object_names(marker = keys[-1])
132
keys += objs
133
return keys
134
except self.client_exc as resperr:
135
log.Warn("Listing of '%s' failed (attempt %s): pyrax returned: %s %s"
136
% (self.container, n, resperr.__class__.__name__, resperr.message))
137
except Exception as e:
138
log.Warn("Listing of '%s' failed (attempt %s): %s: %s"
139
% (self.container, n, e.__class__.__name__, str(e)))
140
log.Debug("Backtrace of previous error: %s"
141
% exception_traceback())
142
time.sleep(30)
143
log.Warn("Giving up listing of '%s' after %s attempts"
144
% (self.container, globals.num_retries))
145
raise BackendException("Error listing '%s'"
146
% (self.container))
147
148
def delete_one(self, remote_filename):
149
for n in range(1, globals.num_retries + 1):
150
log.Info("Deleting '%s/%s'" % (self.container, remote_filename))
151
try:
152
self.container.delete_object(remote_filename)
153
return
154
except self.client_exc as resperr:
155
if n > 1 and resperr.status == 404:
156
# We failed on a timeout, but delete succeeded on the server
157
log.Warn("Delete of '%s' missing after retry - must have succeded earler" % remote_filename)
158
return
159
log.Warn("Delete of '%s' failed (attempt %s): pyrax returned: %s %s"
160
% (remote_filename, n, resperr.__class__.__name__, resperr.message))
161
except Exception as e:
162
log.Warn("Delete of '%s' failed (attempt %s): %s: %s"
163
% (remote_filename, n, e.__class__.__name__, str(e)))
164
log.Debug("Backtrace of previous error: %s"
165
% exception_traceback())
166
time.sleep(30)
167
log.Warn("Giving up deleting '%s' after %s attempts"
168
% (remote_filename, globals.num_retries))
169
raise BackendException("Error deleting '%s/%s'"
170
% (self.container, remote_filename))
171
172
def delete(self, filename_list):
173
for file_ in filename_list:
174
self.delete_one(file_)
175
log.Debug("Deleted '%s/%s'" % (self.container, file_))
176
177
@retry
178
def _query_file_info(self, filename, raise_errors = False):
89
# Cloud Files will return a max of 10,000 objects. We have
90
# to make multiple requests to get them all.
91
objs = self.container.get_object_names()
92
keys = objs
93
while len(objs) == 10000:
94
objs = self.container.get_object_names(marker = keys[-1])
95
keys += objs
96
return keys
97
98
def _delete(self, filename):
99
self.container.delete_object(filename)
100
101
def _query(self, filename):
179
102
try:
180
103
sobject = self.container.get_object(filename)
181
104
return {'size': sobject.total_bytes}
182
105
except self.nso_exc:
183
106
return {'size': -1}
184
except Exception as e:
185
log.Warn("Error querying '%s/%s': %s"
186
"" % (self.container,
187
filename,
188
str(e)))
189
if raise_errors:
190
raise e
191
else:
192
return {'size': None}
193
107
194
108
duplicity.backend.register_backend("cf+http", PyraxBackend)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1