« back to all changes in this revision
Viewing changes to GTG/backends/tweepy/cache.py
- browse files at revision 1268
- compare with another revision
- compare with revision 1268
- download diff
- download tarball
- view history from revision 1268
- Committer: Izidor Matušov
- Date: 2013-01-10 15:03:42 UTC
- Revision ID: izidor.matusov@gmail.com-20130110150342-ajwnwmc2trh9ia2v
Removing broken twitter and tweepy services
I don't know anybody uses them. They are broken, and pretty artificial (more proof of the concept). We should focus our efforts on normal synchronization.
I don't know anybody uses them. They are broken, and pretty artificial (more proof of the concept). We should focus our efforts on normal synchronization.
- files removed:
- GTG/backends/backend_identica.py
- GTG/backends/tweepy
added
removed
GTG/backends/tweepy/cache.py
1
# Tweepy
2
# Copyright 2009-2010 Joshua Roesslein
3
# See LICENSE for details.
4
5
import time
6
import threading
7
import os
8
import cPickle as pickle
9
10
try:
11
import hashlib
12
except ImportError:
13
# python 2.4
14
import md5 as hashlib
15
16
try:
17
import fcntl
18
except ImportError:
19
# Probably on a windows system
20
# TODO: use win32file
21
pass
22
23
24
class Cache(object):
25
"""Cache interface"""
26
27
def __init__(self, timeout=60):
28
"""Initialize the cache
29
timeout: number of seconds to keep a cached entry
30
"""
31
self.timeout = timeout
32
33
def store(self, key, value):
34
"""Add new record to cache
35
key: entry key
36
value: data of entry
37
"""
38
raise NotImplementedError
39
40
def get(self, key, timeout=None):
41
"""Get cached entry if exists and not expired
42
key: which entry to get
43
timeout: override timeout with this value [optional]
44
"""
45
raise NotImplementedError
46
47
def count(self):
48
"""Get count of entries currently stored in cache"""
49
raise NotImplementedError
50
51
def cleanup(self):
52
"""Delete any expired entries in cache."""
53
raise NotImplementedError
54
55
def flush(self):
56
"""Delete all cached entries"""
57
raise NotImplementedError
58
59
60
class MemoryCache(Cache):
61
"""In-memory cache"""
62
63
def __init__(self, timeout=60):
64
Cache.__init__(self, timeout)
65
self._entries = {}
66
self.lock = threading.Lock()
67
68
def __getstate__(self):
69
# pickle
70
return {'entries': self._entries, 'timeout': self.timeout}
71
72
def __setstate__(self, state):
73
# unpickle
74
self.lock = threading.Lock()
75
self._entries = state['entries']
76
self.timeout = state['timeout']
77
78
def _is_expired(self, entry, timeout):
79
return timeout > 0 and (time.time() - entry[0]) >= timeout
80
81
def store(self, key, value):
82
self.lock.acquire()
83
self._entries[key] = (time.time(), value)
84
self.lock.release()
85
86
def get(self, key, timeout=None):
87
self.lock.acquire()
88
try:
89
# check to see if we have this key
90
entry = self._entries.get(key)
91
if not entry:
92
# no hit, return nothing
93
return None
94
95
# use provided timeout in arguments if provided
96
# otherwise use the one provided during init.
97
if timeout is None:
98
timeout = self.timeout
99
100
# make sure entry is not expired
101
if self._is_expired(entry, timeout):
102
# entry expired, delete and return nothing
103
del self._entries[key]
104
return None
105
106
# entry found and not expired, return it
107
return entry[1]
108
finally:
109
self.lock.release()
110
111
def count(self):
112
return len(self._entries)
113
114
def cleanup(self):
115
self.lock.acquire()
116
try:
117
for k, v in self._entries.items():
118
if self._is_expired(v, self.timeout):
119
del self._entries[k]
120
finally:
121
self.lock.release()
122
123
def flush(self):
124
self.lock.acquire()
125
self._entries.clear()
126
self.lock.release()
127
128
129
class FileCache(Cache):
130
"""File-based cache"""
131
132
# locks used to make cache thread-safe
133
cache_locks = {}
134
135
def __init__(self, cache_dir, timeout=60):
136
Cache.__init__(self, timeout)
137
if os.path.exists(cache_dir) is False:
138
os.mkdir(cache_dir)
139
self.cache_dir = cache_dir
140
if cache_dir in FileCache.cache_locks:
141
self.lock = FileCache.cache_locks[cache_dir]
142
else:
143
self.lock = threading.Lock()
144
FileCache.cache_locks[cache_dir] = self.lock
145
146
if os.name == 'posix':
147
self._lock_file = self._lock_file_posix
148
self._unlock_file = self._unlock_file_posix
149
elif os.name == 'nt':
150
self._lock_file = self._lock_file_win32
151
self._unlock_file = self._unlock_file_win32
152
else:
153
print 'Warning! FileCache locking not supported on this system!'
154
self._lock_file = self._lock_file_dummy
155
self._unlock_file = self._unlock_file_dummy
156
157
def _get_path(self, key):
158
md5 = hashlib.md5()
159
md5.update(key)
160
return os.path.join(self.cache_dir, md5.hexdigest())
161
162
def _lock_file_dummy(self, path, exclusive=True):
163
return None
164
165
def _unlock_file_dummy(self, lock):
166
return
167
168
def _lock_file_posix(self, path, exclusive=True):
169
lock_path = path + '.lock'
170
if exclusive is True:
171
f_lock = open(lock_path, 'w')
172
fcntl.lockf(f_lock, fcntl.LOCK_EX)
173
else:
174
f_lock = open(lock_path, 'r')
175
fcntl.lockf(f_lock, fcntl.LOCK_SH)
176
if os.path.exists(lock_path) is False:
177
f_lock.close()
178
return None
179
return f_lock
180
181
def _unlock_file_posix(self, lock):
182
lock.close()
183
184
def _lock_file_win32(self, path, exclusive=True):
185
# TODO: implement
186
return None
187
188
def _unlock_file_win32(self, lock):
189
# TODO: implement
190
return
191
192
def _delete_file(self, path):
193
os.remove(path)
194
if os.path.exists(path + '.lock'):
195
os.remove(path + '.lock')
196
197
def store(self, key, value):
198
path = self._get_path(key)
199
self.lock.acquire()
200
try:
201
# acquire lock and open file
202
f_lock = self._lock_file(path)
203
datafile = open(path, 'wb')
204
205
# write data
206
pickle.dump((time.time(), value), datafile)
207
208
# close and unlock file
209
datafile.close()
210
self._unlock_file(f_lock)
211
finally:
212
self.lock.release()
213
214
def get(self, key, timeout=None):
215
return self._get(self._get_path(key), timeout)
216
217
def _get(self, path, timeout):
218
if os.path.exists(path) is False:
219
# no record
220
return None
221
self.lock.acquire()
222
try:
223
# acquire lock and open
224
f_lock = self._lock_file(path, False)
225
datafile = open(path, 'rb')
226
227
# read pickled object
228
created_time, value = pickle.load(datafile)
229
datafile.close()
230
231
# check if value is expired
232
if timeout is None:
233
timeout = self.timeout
234
if timeout > 0 and (time.time() - created_time) >= timeout:
235
# expired! delete from cache
236
value = None
237
self._delete_file(path)
238
239
# unlock and return result
240
self._unlock_file(f_lock)
241
return value
242
finally:
243
self.lock.release()
244
245
def count(self):
246
c = 0
247
for entry in os.listdir(self.cache_dir):
248
if entry.endswith('.lock'):
249
continue
250
c += 1
251
return c
252
253
def cleanup(self):
254
for entry in os.listdir(self.cache_dir):
255
if entry.endswith('.lock'):
256
continue
257
self._get(os.path.join(self.cache_dir, entry), None)
258
259
def flush(self):
260
for entry in os.listdir(self.cache_dir):
261
if entry.endswith('.lock'):
262
continue
263
self._delete_file(os.path.join(self.cache_dir, entry))
264
Loggerhead is a web-based interface for Breezy
Version: 2.0.1