1
# Copyright 2002 Ben Escoto
3
# This file is part of duplicity.
5
# duplicity is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
8
# 02139, USA; either version 2 of the License, or (at your option) any
9
# later version; incorporated herein by reference.
11
"""Miscellaneous classes and methods"""
16
class MiscError(Exception):
17
"""Signifies a miscellaneous error..."""
21
class FileVolumeWriter:
22
"""Split up an incoming fileobj into multiple volumes on disk
24
This class can also be used as an iterator. It returns the
25
filenames of the files it writes.
28
volume_size = 50 * 1024 * 1024
30
def __init__(self, infp, file_prefix):
31
"""FileVolumeWriter initializer
33
infp is a file object opened for reading. It will be closed
34
at end. file_prefix is the full path of the volumes that will
35
be written. If more than one is required, it will be appended
40
self.prefix = file_prefix
41
self.current_index = 1
42
self.finished = None # set to true when completely done
43
self.buffer = "" # holds data that belongs in next volume
45
def get_initial_buf(self):
46
"""Get first value of buffer, from self.buffer or infp"""
51
else: return self.infp.read(self.blocksize)
53
def write_volume(self, outfp):
54
"""Write self.volume_size bytes from self.infp to outfp
56
Return None if we have reached end of infp without reaching
57
volume size, and false otherwise.
60
bytes_written, buf = 0, self.get_initial_buf()
61
while len(buf) + bytes_written <= self.volume_size:
62
if not buf: # reached end of input
65
if len(buf) + bytes_written > self.volume_size: break
67
bytes_written += len(buf)
68
buf = self.infp.read(self.blocksize)
70
remainder = self.volume_size - bytes_written
71
assert remainder < len(buf)
72
outfp.write(buf[:remainder])
74
self.buffer = buf[remainder:]
78
"""Write next file, return filename"""
79
if self.finished: raise StopIteration
81
filename = "%s.%d" % (self.prefix, self.current_index)
82
log.Log("Starting to write %s" % filename, 5)
83
outfp = open(filename, "wb")
85
if not self.write_volume(outfp): # end of input
87
if self.current_index == 1: # special case first index
88
log.Log("One only volume required.\n"
89
"Renaming %s to %s" % (filename, self.prefix), 4)
90
os.rename(filename, self.prefix)
92
else: self.current_index += 1
95
def __iter__(self): return self
99
"""Buffer file open for reading, so reads will happen in fixed sizes
101
This is currently used to buffer a GzipFile, because that class
102
apparently doesn't respond well to arbitrary read sizes.
105
def __init__(self, fileobj, blocksize = 32 * 1024):
106
self.fileobj = fileobj
108
self.blocksize = blocksize
110
def read(self, length = -1):
111
"""Return length bytes, or all if length < 0"""
114
buf = self.fileobj.read(self.blocksize)
117
real_length = len(self.buffer)
119
while len(self.buffer) < length:
120
buf = self.fileobj.read(self.blocksize)
123
real_length = min(length, len(self.buffer))
124
result = self.buffer[:real_length]
125
self.buffer = self.buffer[real_length:]
128
def close(self): self.fileobj.close()
131
def copyfileobj(infp, outfp, byte_count = -1):
132
"""Copy byte_count bytes from infp to outfp, or all if byte_count < 0
134
Returns the number of bytes actually written (may be less than
135
byte_count if find eof. Does not close either fileobj.
138
blocksize = 32 * 1024
142
buf = infp.read(blocksize)
144
bytes_written += len(buf)
147
while bytes_written + blocksize <= byte_count:
148
buf = infp.read(blocksize)
150
bytes_written += len(buf)
152
buf = infp.read(byte_count - bytes_written)
153
bytes_written += len(buf)
157
def copyfileobj_close(infp, outfp):
158
"""Copy infp to outfp, closing afterwards"""
159
copyfileobj(infp, outfp)
160
if infp.close(): raise MiscError("Error closing input file")
161
if outfp.close(): raise MiscError("Error closing output file")
added
removed
duplicity/misc.py
