Source code for duplicity.backends.copycombackend

# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; coding: utf-8 -*-
#
# Copyright 2014 Marco Trevisan (TreviƱo) <mail@3v1n0.net>
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import os.path
import sys

import duplicity.backend
from duplicity import log
from duplicity.errors import BackendException


[docs]class CoPyCloud: API_URI = 'https://api.copy.com' DEFAULT_ENCODING = 'latin-1' DEFAULT_HEADERS = {'X-Client-Type': 'api', 'X-Api-Version': '1.0', 'X-Authorization': '', 'Accept': 'application/json'} PART_MAX_SIZE = 1024 * 1024 PARTS_HEADER_FMT = '!IIIIII' PARTS_HEADER_SIG = 0xba5eba11 PARTS_HEADER_VERSION = 1 PART_ITEM_FMT = '!IIII73sIIII' PART_ITEM_SIG = 0xcab005e5 PART_ITEM_VERSION = 1
[docs] class Error(Exception): def __init__(self, message): Exception.__init__(self, message)
def __init__(self, username, password): import urllib3 self.http = urllib3.connection_from_url(self.API_URI, block=True, maxsize=1) res = self.__post_req('auth_user', {'username': username, 'password': password}) if not res or 'auth_token' not in res: raise CoPyCloud.Error("Invalid Login") self.DEFAULT_HEADERS['X-Authorization'] = res['auth_token'].encode('ascii', 'ignore') def __req(self, req_type, method, params={}, headers={}): import json headers.update(self.DEFAULT_HEADERS) method = '/' + method if method[0] != '/' else method if isinstance(params, dict): res = self.http.request_encode_body(req_type, method, {'data': json.dumps(params)}, headers, encode_multipart=False) else: res = self.http.urlopen(req_type, method, params, headers) if res.status != 200: raise CoPyCloud.Error("Got HTTP error " + str(res.status)) try: if 'content-type' in res.headers and res.headers['content-type'] == 'application/json' \ or res.data.startswith('{'): jd = json.loads(res.data.decode(self.DEFAULT_ENCODING), self.DEFAULT_ENCODING) if jd and 'result' in jd and jd['result'] == 'error': raise CoPyCloud.Error("Error %s: %s" % (jd['error_code'], jd['error_string'])) return jd except ValueError: pass return res.data def __post_req(self, method, params={}, headers={}): return self.__req('POST', method, params, headers) def __get_req(self, method, headers={}): return self.__req('GET', method, headers) def __binary_parts_req(self, method, parts, share_id=0, headers={}): if not len(parts): return import struct invalid_parts = [] header_size = struct.calcsize(self.PARTS_HEADER_FMT) item_base_size = struct.calcsize(self.PART_ITEM_FMT) items_data_size = sum([p['size'] if 'data' in p else 0 for p in parts]) buf = bytearray(header_size + item_base_size * len(parts) + items_data_size) error_code = 0 padding = 0 pos = 0 struct.pack_into(self.PARTS_HEADER_FMT, buf, pos, self.PARTS_HEADER_SIG, header_size, self.PARTS_HEADER_VERSION, len(buf) - header_size, len(parts), error_code) pos += header_size for part in parts: data_size = part['size'] if 'data' in part else 0 part_size = item_base_size + data_size fingerprint = bytes(part['fingerprint'].encode(self.DEFAULT_ENCODING)) struct.pack_into(self.PART_ITEM_FMT, buf, pos, self.PART_ITEM_SIG, part_size, self.PART_ITEM_VERSION, share_id, fingerprint, part['size'], data_size, error_code, padding) pos += item_base_size if data_size > 0: buf[pos:pos + data_size] = part['data'] pos += data_size ret = self.__post_req(method, buf, {'Content-Type': 'application/octet-stream'}) pos = 0 r = (sig, header_size, version, parts_size, parts_num, error) = \ struct.unpack_from(self.PARTS_HEADER_FMT, ret, pos) pos += header_size if sig != self.PARTS_HEADER_SIG: raise CoPyCloud.Error("Invalid binary header signature from server") if error != 0: raise CoPyCloud.Error("Invalid binary response from server: " + str(ret[pos:])) if header_size != struct.calcsize(self.PARTS_HEADER_FMT): raise CoPyCloud.Error("Invalid binary header size from server") if version != self.PARTS_HEADER_VERSION: raise CoPyCloud.Error("Binary header version mismatch") if parts_num != len(parts): raise CoPyCloud.Error("Part count mismatch") for part in parts: (sig, item_size, version, share_id, fingerprint, remote_size, data_size, error, padding) = \ struct.unpack_from(self.PART_ITEM_FMT, ret, pos) if sig != self.PART_ITEM_SIG: raise CoPyCloud.Error("Invalid binary part item header signature from server") if version != self.PART_ITEM_VERSION: raise CoPyCloud.Error("Binary part item version mismatch") if fingerprint[:-1] != bytes(part['fingerprint'].encode(self.DEFAULT_ENCODING)): raise CoPyCloud.Error("Part %u fingerprint mismatch" % part['offset']) if 'data' in part: if error != 0: offset = pos + item_base_size raise CoPyCloud.Error("Invalid binary part item: " + str(ret[offset:offset + data_size])) if item_size != item_base_size: raise CoPyCloud.Error("Invalid binary part item size received from server") if remote_size != part['size']: raise CoPyCloud.Error("Part %u local/remote size mismatch" % part['offset']) else: if error != 0 or remote_size != part['size']: invalid_parts.append(part) pos += item_base_size return invalid_parts def __update_objects(self, parameters): p = [parameters] if isinstance(parameters, dict) else parameters self.__post_req('update_objects', {'meta': p}) def __sanitize_path(self, path): path = '/' if not path or not len(path) else path return '/' + path if path[0] != '/' else path def __get_file_parts(self, f): import hashlib parts = [] size = os.path.getsize(f.name) while f.tell() < size: offset = f.tell() part_data = f.read(self.PART_MAX_SIZE) fingerprint = hashlib.md5(part_data).hexdigest() + hashlib.sha1(part_data).hexdigest() parts.append({'fingerprint': fingerprint, 'offset': offset, 'size': len(part_data)}) if f.tell() != size: raise CoPyCloud.Error("Impossible to generate full parts for file " + f.name) return parts def __fill_file_parts(self, f, parts): for part in parts: f.seek(part['offset']) part['data'] = f.read(part['size'])
[docs] def list_files(self, path=None, max_items=sys.maxsize): path = path = self.__sanitize_path(path) parameters = {'path': path, 'max_items': max_items} res = self.__post_req('list_objects', parameters) if not res or 'children' not in res: raise CoPyCloud.Error("Impossible to retrieve the files") if 'object' in res and 'type' in res['object'] and res['object']['type'] == 'file': return res['object'] return res['children']
[docs] def remove(self, paths): if isinstance(paths, str): if not len(paths): raise CoPyCloud.Error("Impossible to remove a file with an empty path") paths = [paths] if paths is None: raise CoPyCloud.Error("Impossible to remove files with invalid path") self.__update_objects([{'action': 'remove', 'path': self.__sanitize_path(p)} for p in paths])
[docs] def download(self, path): if not path or not len(path): raise CoPyCloud.Error("Impossible to download a file with an empty path") if not len(self.list_files(path, max_items=1)): raise CoPyCloud.Error("Impossible to download '" + path + "'") return self.__post_req('download_object', {'path': path})
[docs] def upload(self, source, dest, parallel=5, share_id=0): try: f = open(source, 'rb') except Exception as e: raise CoPyCloud.Error("Impossible to open source file " + str(e)) parts = self.__get_file_parts(f) parts_chunks = [parts[i:i + parallel] for i in range(0, len(parts), parallel)] for parts_chunk in parts_chunks: missing_parts = self.__binary_parts_req('has_object_parts', parts_chunk) if len(missing_parts): self.__fill_file_parts(f, missing_parts) self.__binary_parts_req('send_object_parts', missing_parts) for part in parts_chunk: del(part['data']) update_params = {'action': 'create', 'object_type': 'file', 'path': self.__sanitize_path(dest), 'size': os.path.getsize(f.name), 'parts': parts} self.__update_objects(update_params) f.close()
[docs]class CopyComBackend(duplicity.backend.Backend): """Copy.com duplicity backend""" def __init__(self, parsed_url): """Connect to Copy.com""" duplicity.backend.Backend.__init__(self, parsed_url) try: self.copy = CoPyCloud(parsed_url.username, self.get_password()) except CoPyCloud.Error as e: raise BackendException(e) [self.folder] = parsed_url.path[1:].split('/') def _list(self): """List files in folder""" try: return [os.path.basename(f['path']) for f in self.copy.list_files(self.folder)] except CoPyCloud.Error as e: raise BackendException(e) def _query(self, filename): try: file_info = self.copy.list_files(os.path.join(self.folder, filename)) return {'size': int(file_info['size']) if 'size' in file_info else -1} except CoPyCloud.Error as e: raise BackendException(e) def _put(self, source_path, remote_filename): """Upload local file to cloud""" try: self.copy.upload(source_path.get_canonical(), os.path.join(self.folder, remote_filename)) except CoPyCloud.Error as e: raise BackendException(e) def _get(self, remote_filename, local_path): """Save cloud file locally""" try: raw = self.copy.download(os.path.join(self.folder, remote_filename)) except CoPyCloud.Error as e: raise BackendException(e) f = local_path.open(mode='wb') f.write(raw) f.close() def _delete(self, filename): """Delete a file""" try: self.copy.remove(os.path.join(self.folder, filename)) except CoPyCloud.Error as e: raise BackendException(e)
''' This must be disabled here, because if a file in list does not exist, the Copy server will stop deleting the subsequent stuff, raising an error, making test_delete_list to fail. def _delete_list(self, filenames): """Delete list of files""" try: self.copy.remove([os.path.join(self.folder, f) for f in filenames]) except CoPyCloud.Error as e: if 'Error 1024' in e: pass # "Ignore file can't be located error, in this case" ''' duplicity.backend.register_backend('copy', CopyComBackend) duplicity.backend.uses_netloc.extend(['copy'])