1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
|
#!/usr/bin/env python
"""Put a S3 bucket on a diet by removing old files. Used to delete
old builds from the cbuild archive.
"""
import collections
import sys
import datetime
import re
import os.path
import ConfigParser
import logging
import simples3
import hurry.filesize
Entry = collections.namedtuple('Entry', 'key timestamp etag size basename')
def main():
dry_run = False
logging.basicConfig(level=logging.INFO)
config = ConfigParser.SafeConfigParser()
config.read(os.path.expanduser('~/.s3cfg'))
bucket = simples3.S3Bucket(sys.argv[1],
access_key=config.get('default', 'access_key'),
secret_key=config.get('default', 'secret_key'))
by_key = {}
entries = []
for v in bucket.listdir():
entry = Entry(*(v + (os.path.basename(v[0]), )))
by_key[entry.key] = entry
entries.append(entry)
# Do some linting
to_delete = []
# Delete the signature if the original file is gone
for entry in entries:
if entry.key.endswith('.asc'):
original = entry.key.replace('.asc', '')
if original not in by_key:
logging.debug('Deleting orphan signature %s' % entry.basename)
to_delete.append(entry)
# Delete anything that's too old
now = datetime.datetime.utcnow()
for entry in entries:
age = now - entry.timestamp
is_merge = '+bzr' in entry.basename and len(entry.basename.split('~')) == 3
if re.search(r'-201.\.\d\d', entry.basename) and '+bzr' not in entry.basename:
logging.debug("Not deleting %s as it's special" % entry.basename)
elif is_merge and age.days >= 30:
logging.debug('Deleting %d day old merge %s' % (age.days, entry.basename))
to_delete.append(entry)
elif age.days >= 45:
logging.debug('Deleting %d day old %s' % (age.days, entry.basename))
to_delete.append(entry)
else:
pass
# Nuke any duplicates
to_delete = sorted(set(to_delete))
total = sum(x.size for x in entries)
deleting = sum(x.size for x in to_delete)
logging.info('Total stored: %s in %d entries' % (hurry.filesize.size(total), len(entries)))
logging.info('To delete: %s in %d entries' % (hurry.filesize.size(deleting), len(to_delete)))
for entry in sorted(to_delete, key=lambda x: -x.size):
logging.info('Deleting %s' % entry.basename)
if not dry_run:
del bucket[entry.key]
if __name__ == '__main__':
main()
|