1
# Copyright (c) 2010-2013 OpenStack, LLC.
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
7
# http://www.apache.org/licenses/LICENSE-2.0
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
16
"""General object server functions."""
18
import cPickle as pickle
25
from os.path import basename, dirname, join
27
from swift.common.exceptions import PathNotDir
28
from swift.common.utils import lock_path, renamer, write_pickle
33
HASH_FILE = 'hashes.pkl'
36
def quarantine_renamer(device_path, corrupted_file_path):
38
In the case that a file is corrupted, move it to a quarantined
39
area to allow replication to fix it.
41
:params device_path: The path to the device the corrupted file is on.
42
:params corrupted_file_path: The path to the file you want quarantined.
44
:returns: path (str) of directory the file was moved to
45
:raises OSError: re-raises non errno.EEXIST / errno.ENOTEMPTY
46
exceptions from rename
48
from_dir = dirname(corrupted_file_path)
49
to_dir = join(device_path, 'quarantined', 'objects', basename(from_dir))
50
invalidate_hash(dirname(from_dir))
52
renamer(from_dir, to_dir)
54
if e.errno not in (errno.EEXIST, errno.ENOTEMPTY):
56
to_dir = "%s-%s" % (to_dir, uuid.uuid4().hex)
57
renamer(from_dir, to_dir)
61
def hash_suffix(path, reclaim_age):
63
Performs reclamation and returns an md5 of all (remaining) files.
65
:param reclaim_age: age in seconds at which to remove tombstones
66
:raises PathNotDir: if given path is not a valid directory
67
:raises OSError: for non-ENOTDIR errors
71
path_contents = sorted(os.listdir(path))
73
if err.errno in (errno.ENOTDIR, errno.ENOENT):
76
for hsh in path_contents:
77
hsh_path = join(path, hsh)
79
files = os.listdir(hsh_path)
81
if err.errno == errno.ENOTDIR:
82
partition_path = dirname(path)
83
objects_path = dirname(partition_path)
84
device_path = dirname(objects_path)
85
quar_path = quarantine_renamer(device_path, hsh_path)
87
_('Quarantined %s to %s because it is not a directory') %
88
(hsh_path, quar_path))
92
if files[0].endswith('.ts'):
93
# remove tombstones older than reclaim_age
94
ts = files[0].rsplit('.', 1)[0]
95
if (time.time() - float(ts)) > reclaim_age:
96
os.unlink(join(hsh_path, files[0]))
97
files.remove(files[0])
99
files.sort(reverse=True)
100
meta = data = tomb = None
101
for filename in list(files):
102
if not meta and filename.endswith('.meta'):
104
if not data and filename.endswith('.data'):
106
if not tomb and filename.endswith('.ts'):
108
if (filename < tomb or # any file older than tomb
109
filename < data or # any file older than data
110
(filename.endswith('.meta') and
111
filename < meta)): # old meta
112
os.unlink(join(hsh_path, filename))
113
files.remove(filename)
116
for filename in files:
122
return md5.hexdigest()
125
def invalidate_hash(suffix_dir):
127
Invalidates the hash for a suffix_dir in the partition's hashes file.
129
:param suffix_dir: absolute path to suffix dir whose hash needs
133
suffix = os.path.basename(suffix_dir)
134
partition_dir = os.path.dirname(suffix_dir)
135
hashes_file = join(partition_dir, HASH_FILE)
136
with lock_path(partition_dir):
138
with open(hashes_file, 'rb') as fp:
139
hashes = pickle.load(fp)
140
if suffix in hashes and not hashes[suffix]:
144
hashes[suffix] = None
145
write_pickle(hashes, hashes_file, partition_dir, PICKLE_PROTOCOL)
148
def get_hashes(partition_dir, recalculate=None, do_listdir=False,
149
reclaim_age=ONE_WEEK):
151
Get a list of hashes for the suffix dir. do_listdir causes it to mistrust
152
the hash cache for suffix existence at the (unexpectedly high) cost of a
153
listdir. reclaim_age is just passed on to hash_suffix.
155
:param partition_dir: absolute path of partition to get hashes for
156
:param recalculate: list of suffixes which should be recalculated when got
157
:param do_listdir: force existence check for all hashes in the partition
158
:param reclaim_age: age at which to remove tombstones
160
:returns: tuple of (number of suffix dirs hashed, dictionary of hashes)
164
hashes_file = join(partition_dir, HASH_FILE)
166
force_rewrite = False
170
if recalculate is None:
174
with open(hashes_file, 'rb') as fp:
175
hashes = pickle.load(fp)
176
mtime = os.path.getmtime(hashes_file)
181
for suff in os.listdir(partition_dir):
183
hashes.setdefault(suff, None)
185
hashes.update((hash_, None) for hash_ in recalculate)
186
for suffix, hash_ in hashes.items():
188
suffix_dir = join(partition_dir, suffix)
190
hashes[suffix] = hash_suffix(suffix_dir, reclaim_age)
195
logging.exception(_('Error hashing suffix'))
198
with lock_path(partition_dir):
199
if force_rewrite or not os.path.exists(hashes_file) or \
200
os.path.getmtime(hashes_file) == mtime:
202
hashes, hashes_file, partition_dir, PICKLE_PROTOCOL)
203
return hashed, hashes
204
return get_hashes(partition_dir, recalculate, do_listdir,
207
return hashed, hashes
added
removed
swift/obj/base.py
