comparison MoinMoin/support/werkzeug/contrib/cache.py @ 4301:c689dfa55de1

Import werkzeug 0.3.1 package into MoinMoin.support (still need to insert CREDITS)
author Florian Krupicka <florian.krupicka@googlemail.com>
date Wed, 13 Aug 2008 21:05:42 +0200
parents
children 246ba4eecab2
comparison
equal deleted inserted replaced
4300:867da7983aba 4301:c689dfa55de1
1 # -*- coding: utf-8 -*-
2 """
3 werkzeug.contrib.cache
4 ~~~~~~~~~~~~~~~~~~~~~~
5
6 Small helper module that provides a simple interface to memcached, a
7 simple django-inspired in-process cache and a file system based cache.
8
9 The idea is that it's possible to switch caching systems without changing
10 much code in the application.
11
12
13 :copyright: 2007-2008 by Armin Ronacher.
14 :license: BSD, see LICENSE for more details.
15 """
16 import os
17 import re
18 try:
19 from hashlib import md5
20 except ImportError:
21 from md5 import new as md5
22 from itertools import izip
23 from time import time
24 from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL
25
26 have_memcache = True
27 try:
28 import cmemcache as memcache
29 is_cmemcache = True
30 except ImportError:
31 try:
32 import memcache
33 is_cmemcache = False
34 except ImportError:
35 have_memcache = False
36
37
38 class BaseCache(object):
39 """Baseclass for the cache systems."""
40
41 def __init__(self, default_timeout=300):
42 self.default_timeout = default_timeout
43
44 def get(self, key):
45 return None
46 delete = get
47
48 def get_many(self, *keys):
49 return map(self.get, keys)
50
51 def get_dict(self, *keys):
52 return dict(izip(keys, self.get_many(keys)))
53
54 def set(self, key, value, timeout=None):
55 pass
56 add = set
57
58 def set_many(self, mapping, timeout=None):
59 for key, value in mapping.iteritems():
60 self.set(key, value, timeout)
61
62 def delete_many(self, *keys):
63 for key in keys:
64 self.delete(key)
65
66 def clear(self):
67 pass
68
69 def inc(self, key, delta=1):
70 self.set(key, (self.get(key) or 0) + delta)
71
72 def dec(self, key, delta=1):
73 self.set(key, (self.get(key) or 0) - delta)
74
75
76 class NullCache(BaseCache):
77 """A cache that doesn't cache."""
78
79
80 class SimpleCache(BaseCache):
81 """Simple memory cache for single process environments. This class exists
82 mainly for the development server and is not 100% thread safe. It tries
83 to use as many atomic operations as possible and no locks for simplicity
84 but it could happen under heavy load that keys are added multiple times.
85 """
86
87 def __init__(self, threshold=500, default_timeout=300):
88 BaseCache.__init__(self, default_timeout)
89 self._cache = {}
90 self.clear = self._cache.clear
91 self._threshold = threshold
92
93 def _prune(self):
94 if len(self._cache) > self._threshold:
95 now = time()
96 for idx, (key, (expires, _)) in enumerate(self._cache.items()):
97 if expires <= now or idx % 3 == 0:
98 self._cache.pop(key, None)
99
100 def get(self, key):
101 now = time()
102 expires, value = self._cache.get(key, (0, None))
103 if expires > time():
104 return loads(value)
105
106 def set(self, key, value, timeout=None):
107 if timeout is None:
108 timeout = self.default_timeout
109 self._prune()
110 self._cache[key] = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
111
112 def add(self, key, value, timeout=None):
113 if timeout is None:
114 timeout = self.default_timeout
115 if len(self._cache) > self._threshold:
116 self._prune()
117 item = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
118 self._cache.setdefault(key, item)
119
120 def delete(self, key):
121 self._cache.pop(key, None)
122
123
124 _test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match
125
126 class MemcachedCache(BaseCache):
127 """A cache that uses memcached as backend.
128
129 Implementation notes: This cache backend works around some limitations in
130 memcached to simplify the interface. For example unicode keys are encoded
131 to utf-8 on the fly. Methods such as `get_dict` return the keys in the
132 same format as passed. Furthermore all get methods silently ignore key
133 errors to not cause problems when untrusted user data is passed to the get
134 methods which is often the case in web applications.
135 """
136
137 def __init__(self, servers, default_timeout=300):
138 BaseCache.__init__(self, default_timeout)
139 if not have_memcache:
140 raise RuntimeError('no memcache module found')
141
142 # cmemcache has a bug that debuglog is not defined for the
143 # client. Whenever pickle fails you get a weird AttributError.
144 if is_cmemcache:
145 self._client = memcache.Client(map(str, servers))
146 try:
147 self._client.debuglog = lambda *a: None
148 except:
149 pass
150 else:
151 self._client = memcache.Client(servers, False, HIGHEST_PROTOCOL)
152
153 def get(self, key):
154 if isinstance(key, unicode):
155 key = key.encode('utf-8')
156 # memcached doesn't support keys longer than that. Because often
157 # checks for so long keys can occour because it's tested from user
158 # submitted data etc we fail silently for getting.
159 if _test_memcached_key(key):
160 return self._client.get(key)
161
162 def get_dict(self, *keys):
163 key_mapping = {}
164 have_encoded_keys = False
165 for idx, key in enumerate(keys):
166 if isinstance(key, unicode):
167 encoded_key = key.encode('utf-8')
168 have_encoded_keys = True
169 else:
170 encoded_key = key
171 if _test_memcached_key(key):
172 key_mapping[encoded_key] = key
173 # the keys call here is important because otherwise cmemcache
174 # does ugly things. What exaclty I don't know, i think it does
175 # Py_DECREF but quite frankly i don't care.
176 d = rv = self._client.get_multi(key_mapping.keys())
177 if have_encoded_keys:
178 rv = {}
179 for key, value in d.iteritems():
180 rv[key_mapping[key]] = value
181 if len(rv) < len(keys):
182 for key in keys:
183 if key not in rv:
184 rv[key] = None
185 return rv
186
187 def add(self, key, value, timeout=None):
188 if timeout is None:
189 timeout = self.default_timeout
190 if isinstance(key, unicode):
191 key = key.encode('utf-8')
192 self._client.add(key, value, timeout)
193
194 def set(self, key, value, timeout=None):
195 if timeout is None:
196 timeout = self.default_timeout
197 if isinstance(key, unicode):
198 key = key.encode('utf-8')
199 self._client.set(key, value, timeout)
200
201 def get_many(self, *keys):
202 d = self.get_dict(*keys)
203 return [d[key] for key in keys]
204
205 def set_many(self, mapping, timeout=None):
206 if timeout is None:
207 timeout = self.default_timeout
208 new_mapping = {}
209 for key, value in mapping.iteritems():
210 if isinstance(key, unicode):
211 key = key.encode('utf-8')
212 new_mapping[key] = value
213 self._client.set_multi(new_mapping, timeout)
214
215 def delete(self, key):
216 if isinstance(key, unicode):
217 key = key.encode('utf-8')
218 self._client.delete(key)
219
220 def delete_many(self, *keys):
221 keys = list(keys)
222 for idx, key in enumerate(keys):
223 if isinstance(key, unicode):
224 keys[idx] = key.encode('utf-8')
225 self._client.delete_multi(keys)
226
227 def clear(self):
228 self._client.flush_all()
229
230 def inc(self, key, delta=1):
231 if isinstance(key, unicode):
232 key = key.encode('utf-8')
233 self._client.incr(key, key, delta)
234
235 def dec(self, key, delta=1):
236 if isinstance(key, unicode):
237 key = key.encode('utf-8')
238 self._client.decr(key, key, delta)
239
240
241 class FileSystemCache(BaseCache):
242 """A cache that stores the items on the file system."""
243
244 def __init__(self, cache_dir, threshold=500, default_timeout=300):
245 BaseCache.__init__(self, default_timeout)
246 self._path = cache_dir
247 self._threshold = threshold
248 if not os.path.exists(self._path):
249 os.makedirs(self._path)
250
251 def _prune(self):
252 entries = os.listdir(self._path)
253 if len(entries) > self._threshold:
254 now = time()
255 for idx, key in enumerate(entries):
256 try:
257 f = file(self._get_filename(key))
258 if pickle.load(f) > now and idx % 3 != 0:
259 f.close()
260 continue
261 except:
262 f.close()
263 self.delete(key)
264
265 def _get_filename(self, key):
266 hash = md5(key).hexdigest()
267 return os.path.join(self._path, hash)
268
269 def get(self, key):
270 filename = self._get_filename(key)
271 try:
272 f = file(filename, 'rb')
273 try:
274 if load(f) >= time():
275 return load(f)
276 finally:
277 f.close()
278 os.remove(filename)
279 except:
280 return None
281
282 def add(self, key, value, timeout=None):
283 filename = self._get_filename(key)
284 if not os.path.exists(filename):
285 self.set(key, value, timeout)
286
287 def set(self, key, value, timeout=None):
288 if timeout is None:
289 timeout = self.default_timeout
290 filename = self._get_filename(key)
291 self._prune()
292 try:
293 f = file(filename, 'wb')
294 try:
295 dump(int(time() + timeout), f, 1)
296 dump(value, f, HIGHEST_PROTOCOL)
297 finally:
298 f.close()
299 except (IOError, OSError):
300 pass
301
302 def delete(self, key):
303 try:
304 os.remove(self._get_filename(key))
305 except (IOError, OSError):
306 pass
307
308 def clear(self):
309 for key in os.listdir(self._path):
310 self.delete(key)