~mterry/duplicity/backend-unification : revision 972

1

# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-

2

#

3

4

# Authors: Michael Terry <michael.terry@canonical.com>

5

# Alexander Zangerl <az@debian.org>

6

#

7

# This file is part of duplicity.

8

#

9

# Duplicity is free software; you can redistribute it and/or modify it

10

# under the terms of the GNU General Public License as published by the

11

# Free Software Foundation; either version 2 of the License, or (at your

12

# option) any later version.

13

#

14

# Duplicity is distributed in the hope that it will be useful, but

15

# WITHOUT ANY WARRANTY; without even the implied warranty of

16

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

17

# General Public License for more details.

18

#

19

# You should have received a copy of the GNU General Public License

20

# along with duplicity. If not, see <http://www.gnu.org/licenses/>.

21

22

import duplicity.backend

23

from duplicity.errors import BackendException

24

from duplicity import log

25

from duplicity import globals

26

27

from urlparse import urlparse, parse_qsl

28

from json import loads, dumps

29

# python3 splitted urllib

30

try:

31

import urllib

32

except ImportError:

33

import urllib.request as urllib

34

import getpass

35

import os

36

import sys

37

import time

38

39

class OAuthHttpClient(object):

40

"""a simple HTTP client with OAuth added on"""

41

def __init__(self):

42

# lazily import non standard python libs

43

global oauth1, Http

44

from oauthlib import oauth1

45

from httplib2 import Http

46

47

self.consumer_key = None

48

self.consumer_secret = None

49

self.token = None

50

self.token_secret = None

51

self.client = Http()

52

53

def set_consumer(self, consumer_key, consumer_secret):

54

self.consumer_key = consumer_key

55

self.consumer_secret = consumer_secret

56

57

def set_token(self, token, token_secret):

58

self.token = token

59

self.token_secret = token_secret

60

61

def _get_oauth_request_header(self, url, method):

62

"""Get an oauth request header given the token and the url"""

63

client = oauth1.Client(

64

unicode(self.consumer_key),

65

client_secret=unicode(self.consumer_secret),

66

resource_owner_key=unicode(self.token),

67

resource_owner_secret=unicode(self.token_secret))

68

url, headers, body = client.sign(

69

unicode(url),

70

http_method=unicode(method))

71

return [url, headers]

72

73

def request(self, url, method="GET", body=None, headers={}, ignore=None):

74

url, oauth_header = self._get_oauth_request_header(url, method)

75

headers.update(oauth_header)

76

77

for n in range(1, globals.num_retries+1):

78

log.Info("making %s request to %s (attempt %d)" % (method,url,n))

79

try:

80

resp, content = self.client.request(url, method, headers=headers, body=body)

81

except Exception, e:

82

log.Info("request failed, exception %s" % e);

83

log.Debug("Backtrace of previous error: %s"

84

% duplicity.util.exception_traceback())

85

if n == globals.num_retries:

86

log.FatalError("Giving up on request after %d attempts, last exception %s" % (n,e))

87

88

if isinstance(body, file):

89

body.seek(0) # Go to the beginning of the file for the retry

90

91

time.sleep(30)

92

continue

93

94

log.Info("completed request with status %s %s" % (resp.status,resp.reason))

95

oops_id = resp.get('x-oops-id', None)

96

if oops_id:

97

log.Debug("Server Error: method %s url %s Oops-ID %s" % (method, url, oops_id))

98

99

if resp['content-type'] == 'application/json':

100

content = loads(content)

101

102

# were we successful? status either 2xx or code we're told to ignore

103

numcode=int(resp.status)

104

if (numcode>=200 and numcode<300) or (ignore and numcode in ignore):

105

return resp, content

106

107

ecode = log.ErrorCode.backend_error

108

if numcode == 402: # Payment Required

109

ecode = log.ErrorCode.backend_no_space

110

elif numcode == 404:

111

ecode = log.ErrorCode.backend_not_found

112

113

if isinstance(body, file):

114

body.seek(0) # Go to the beginning of the file for the retry

115

116

if n < globals.num_retries:

117

time.sleep(30)

118

119

log.FatalError("Giving up on request after %d attempts, last status %d %s" % (n,numcode,resp.reason),

120

ecode)

121

122

123

def get_and_set_token(self,email, password, hostname):

124

"""Acquire an Ubuntu One access token via OAuth with the Ubuntu SSO service.

125

See https://one.ubuntu.com/developer/account_admin/auth/otherplatforms for details.

126

"""

127

128

# Request new access token from the Ubuntu SSO service

129

self.client.add_credentials(email,password)

130

resp, content = self.client.request('https://login.ubuntu.com/api/1.0/authentications?'

131

+'ws.op=authenticate&token_name=Ubuntu%%20One%%20@%%20%s' % hostname)

132

if resp.status!=200:

133

log.FatalError("Token request failed: Incorrect Ubuntu One credentials",log.ErrorCode.backend_permission_denied)

134

self.client.clear_credentials()

135

136

tokendata=loads(content)

137

self.set_consumer(tokendata['consumer_key'],tokendata['consumer_secret'])

138

self.set_token(tokendata['token'],tokendata['token_secret'])

139

140

# and finally tell Ubuntu One about the token

141

resp, content = self.request('https://one.ubuntu.com/oauth/sso-finished-so-get-tokens/')

142

if resp.status!=200:

143

log.FatalError("Ubuntu One token was not accepted: %s %s" % (resp.status,resp.reason))

144

145

return tokendata

146

147

class U1Backend(duplicity.backend.Backend):

148

"""

149

Backend for Ubuntu One, through the use of the REST API.

150

See https://one.ubuntu.com/developer/ for REST documentation.

151

"""

152

def __init__(self, url):

153

duplicity.backend.Backend.__init__(self, url)

154

155

# u1://dontcare/volname or u1+http:///volname

156

path = self.parsed_url.path.lstrip('/')

157

158

self.api_base = "https://one.ubuntu.com/api/file_storage/v1"

159

self.content_base = "https://files.one.ubuntu.com"

160

161

self.volume_uri = "%s/volumes/~/%s" % (self.api_base, path)

162

self.meta_base = "%s/~/%s/" % (self.api_base, path)

163

164

self.client=OAuthHttpClient();

165

166

if 'FTP_PASSWORD' not in os.environ:

167

sys.stderr.write("No Ubuntu One token found in $FTP_PASSWORD, requesting a new one\n")

168

email=raw_input('Enter Ubuntu One account email: ')

169

password=getpass.getpass("Enter Ubuntu One password: ")

170

hostname=os.uname()[1]

171

172

tokendata = self.client.get_and_set_token(email, password, hostname)

173

tokenstring = "%s:%s:%s:%s" % (tokendata['consumer_key'], tokendata['consumer_secret'],

174

tokendata['token'], tokendata['token_secret'])

175

sys.stderr.write("\nPlease record your new Ubuntu One access token for future use with duplicity:\n"

176

+ "FTP_PASSWORD=%s\n\n" % tokenstring)

177

os.environ['FTP_PASSWORD'] = tokenstring

178

179

(consumer,consumer_secret,token,token_secret) = os.environ['FTP_PASSWORD'].split(':')

180

self.client.set_consumer(consumer, consumer_secret)

181

self.client.set_token(token, token_secret)

182

183

resp, content = self.client.request(self.api_base,ignore=[400,401,403])

184

if resp['status']!='200':

185

log.FatalError("Access failed: Ubuntu One credentials incorrect",

186

log.ErrorCode.user_error)

187

188

# Create volume, but check existence first

189

resp, content = self.client.request(self.volume_uri,ignore=[404])

190

if resp['status']=='404':

191

resp, content = self.client.request(self.volume_uri,"PUT")

192

193

def quote(self, url):

194

return urllib.quote(url, safe="/~").replace(" ","%20")

195

196

def put(self, source_path, remote_filename = None):

197

"""Copy file to remote"""

198

if not remote_filename:

199

remote_filename = source_path.get_filename()

200

remote_full = self.meta_base + self.quote(remote_filename)

201

# check if it exists already, returns existing content_path

202

resp, content = self.client.request(remote_full,ignore=[404])

203

if resp['status']=='404':

204

# put with path returns new content_path

205

resp, content = self.client.request(remote_full,

206

method="PUT",

207

headers = { 'content-type': 'application/json' },

208

body=dumps({"kind":"file"}))

209

elif resp['status']!='200':

210

raise BackendException("access to %s failed, code %s" % (remote_filename, resp['status']))

211

212

assert(content['content_path'] is not None)

213

# content_path allows put of the actual material

214

remote_full = self.content_base + self.quote(content['content_path'])

215

log.Info("uploading file %s to location %s" % (remote_filename, remote_full))

216

217

size = os.path.getsize(source_path.name)

218

fh=open(source_path.name,'rb')

219

220

content_type = 'application/octet-stream'

221

headers = {"Content-Length": str(size),

222

"Content-Type": content_type}

223

resp, content = self.client.request(remote_full,

224

method="PUT",

225

body=fh,

226

headers=headers)

227

fh.close()

228

229

def get(self, filename, local_path):

230

"""Get file and put in local_path (Path object)"""

231

232

# get with path returns content_path

233

remote_full = self.meta_base + self.quote(filename)

234

resp, content = self.client.request(remote_full)

235

236

assert(content['content_path'] is not None)

237

# now we have content_path to access the actual material

238

remote_full = self.content_base + self.quote(content['content_path'])

239

log.Info("retrieving file %s from location %s" % (filename, remote_full))

240

resp, content = self.client.request(remote_full)

241

242

f = open(local_path.name, 'wb')

243

f.write(content)

244

f.close()

245

local_path.setdata()

246

247

def _list(self):

248

"""List files in that directory"""

249

remote_full = self.meta_base + "?include_children=true"

250

resp, content = self.client.request(remote_full)

251

252

filelist = []

253

if 'children' in content:

254

for child in content['children']:

255

path = urllib.unquote(child['path'].lstrip('/'))

256

filelist += [path.encode('utf-8')]

257

return filelist

258

259

def delete(self, filename_list):

260

"""Delete all files in filename list"""

261

import types

262

assert type(filename_list) is not types.StringType

263

264

for filename in filename_list:

265

remote_full = self.meta_base + self.quote(filename)

266

resp, content = self.client.request(remote_full,method="DELETE")

267

268

def _query_file_info(self, filename):

269

"""Query attributes on filename"""

270

remote_full = self.meta_base + self.quote(filename)

271

resp, content = self.client.request(remote_full)

272

273

size = content['size']

274

return {'size': size}

275

276

duplicity.backend.register_backend("u1", U1Backend)

277

duplicity.backend.register_backend("u1+http", U1Backend)