comparison MoinMoin/support/werkzeug/contrib/cache.py @ 4609:246ba4eecab2

updated werkzeug to 0.5.pre20090228
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sat, 28 Feb 2009 00:08:31 +0100
parents c689dfa55de1
children 8de563c487be
comparison
equal deleted inserted replaced
4608:3a79d3ca5a83 4609:246ba4eecab2
1 # -*- coding: utf-8 -*- 1 # -*- coding: utf-8 -*-
2 """ 2 """
3 werkzeug.contrib.cache 3 werkzeug.contrib.cache
4 ~~~~~~~~~~~~~~~~~~~~~~ 4 ~~~~~~~~~~~~~~~~~~~~~~
5 5
6 Small helper module that provides a simple interface to memcached, a 6 The main problem with dynamic Web sites is, well, they're dynamic. Each
7 simple django-inspired in-process cache and a file system based cache. 7 time a user requests a page, the webserver executes a lot of code, queries
8 8 the database, renders templates until the visitor gets the page he sees.
9 The idea is that it's possible to switch caching systems without changing 9
10 much code in the application. 10 This is a lot more expensive than just loading a file from the file system
11 11 and sending it to the visitor.
12 12
13 :copyright: 2007-2008 by Armin Ronacher. 13 For most Web applications, this overhead isn't a big deal but once it
14 becomes, you will be glad to have a cache system in place.
15
16 How Caching Works
17 =================
18
19 Caching is pretty simple. Basically you have a cache object lurking around
20 somewhere that is connected to a remote cache or the file system or
21 something else. When the request comes in you check if the current page
22 is already in the cache and if, you're returning it. Otherwise you generate
23 the page and put it into the cache. (Or a fragment of the page, you don't
24 have to cache the full thing)
25
26 Here a simple example of how to cache a sidebar for a template::
27
28 def get_sidebar(user):
29 identifier = 'sidebar_for/user%d' % user.id
30 value = cache.get(identifier)
31 if value is not None:
32 return value
33 value = generate_sidebar_for(user=user)
34 cache.set(identifier, value, timeout=60 * 5)
35 return value
36
37 Creating a Cache Object
38 =======================
39
40 To create a cache object you just import the cache system of your choice
41 from the cache module and instanciate it. Then you can start working
42 with that object:
43
44 >>> from werkzeug.contrib.cache import SimpleCache
45 >>> c = SimpleCache()
46 >>> c.set("foo", "value")
47 >>> c.get("foo")
48 'value'
49 >>> c.get("missing") is None
50 True
51
52 Please keep in mind that you have to create the cache and put it somewhere
53 you have access to it (either as a module global you can import or if you
54 put it onto your WSGI application).
55
56 :copyright: (c) 2009 by the Werkzeug Team, see AUTHORS for more details.
14 :license: BSD, see LICENSE for more details. 57 :license: BSD, see LICENSE for more details.
15 """ 58 """
16 import os 59 import os
17 import re 60 import re
18 try: 61 try:
21 from md5 import new as md5 64 from md5 import new as md5
22 from itertools import izip 65 from itertools import izip
23 from time import time 66 from time import time
24 from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL 67 from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL
25 68
26 have_memcache = True
27 try:
28 import cmemcache as memcache
29 is_cmemcache = True
30 except ImportError:
31 try:
32 import memcache
33 is_cmemcache = False
34 except ImportError:
35 have_memcache = False
36
37 69
38 class BaseCache(object): 70 class BaseCache(object):
39 """Baseclass for the cache systems.""" 71 """Baseclass for the cache systems. All the cache systems implement this
72 API or a superset of it.
73
74 :param default_timeout: the default timeout that is used if no timeout is
75 specified on :meth:`set`.
76 """
40 77
41 def __init__(self, default_timeout=300): 78 def __init__(self, default_timeout=300):
42 self.default_timeout = default_timeout 79 self.default_timeout = default_timeout
43 80
44 def get(self, key): 81 def get(self, key):
82 """Looks up key in the cache and returns it. If the key does not
83 exist `None` is returned instead.
84
85 :param key: the key to be looked up.
86 """
45 return None 87 return None
46 delete = get 88
89 def delete(self, key):
90 """Deletes `key` from the cache. If it does not exist in the cache
91 nothing happens.
92
93 :param key: the key to delete.
94 """
95 pass
47 96
48 def get_many(self, *keys): 97 def get_many(self, *keys):
98 """Returns a list of keys. For each key a item in the list is
99 created. Example::
100
101 foo, bar = cache.get_many("foo", "bar")
102
103 If a key can't be looked up `None` is returned for that key
104 instead.
105
106 :param keys: The function accepts multiple keys as positional
107 arguments.
108 """
49 return map(self.get, keys) 109 return map(self.get, keys)
50 110
51 def get_dict(self, *keys): 111 def get_dict(self, *keys):
52 return dict(izip(keys, self.get_many(keys))) 112 """Works like :meth:`get_many` but returns a dict::
113
114 d = cache.get_dict("foo", "bar")
115 foo = d["foo"]
116 bar = d["bar"]
117
118 :param keys: The function accepts multiple keys as positional
119 arguments.
120 """
121 return dict(izip(keys, self.get_many(*keys)))
53 122
54 def set(self, key, value, timeout=None): 123 def set(self, key, value, timeout=None):
124 """Adds or overrides a key in the cache.
125
126 :param key: the key to set
127 :param value: the value for the key
128 :param timeout: the cache timeout for the key or the default
129 timeout if not specified.
130 """
55 pass 131 pass
56 add = set 132
133 def add(self, key, value, timeout=None):
134 """Works like :meth:`set` but does not override already existing
135 values.
136
137 :param key: the key to set
138 :param value: the value for the key
139 :param timeout: the cache timeout for the key or the default
140 timeout if not specified.
141 """
142 pass
57 143
58 def set_many(self, mapping, timeout=None): 144 def set_many(self, mapping, timeout=None):
145 """Sets multiple keys and values from a dict.
146
147 :param mapping: a dict with the values to set.
148 :param timeout: the cache timeout for the key or the default
149 timeout if not specified.
150 """
59 for key, value in mapping.iteritems(): 151 for key, value in mapping.iteritems():
60 self.set(key, value, timeout) 152 self.set(key, value, timeout)
61 153
62 def delete_many(self, *keys): 154 def delete_many(self, *keys):
155 """Deletes multiple keys at once.
156
157 :param keys: The function accepts multiple keys as positional
158 arguments.
159 """
63 for key in keys: 160 for key in keys:
64 self.delete(key) 161 self.delete(key)
65 162
66 def clear(self): 163 def clear(self):
164 """Clears the cache. Keep in mind that not all caches support
165 clearning of the full cache.
166 """
67 pass 167 pass
68 168
69 def inc(self, key, delta=1): 169 def inc(self, key, delta=1):
170 """Increments the value of a key by `delta`. If the key does
171 not yet exist it is initialized with `delta`.
172
173 For supporting caches this is an atomic operation.
174
175 :param key: the key to increment.
176 :param delta: the delta to add.
177 """
70 self.set(key, (self.get(key) or 0) + delta) 178 self.set(key, (self.get(key) or 0) + delta)
71 179
72 def dec(self, key, delta=1): 180 def dec(self, key, delta=1):
181 """Decrements the value of a key by `delta`. If the key does
182 not yet exist it is initialized with `-delta`.
183
184 For supporting caches this is an atomic operation.
185
186 :param key: the key to increment.
187 :param delta: the delta to subtract.
188 """
73 self.set(key, (self.get(key) or 0) - delta) 189 self.set(key, (self.get(key) or 0) - delta)
74 190
75 191
76 class NullCache(BaseCache): 192 class NullCache(BaseCache):
77 """A cache that doesn't cache.""" 193 """A cache that doesn't cache. This can be useful for unit testing.
194
195 :param default_timeout: a dummy parameter that is ignored but exists
196 for API compatibility with other caches.
197 """
78 198
79 199
80 class SimpleCache(BaseCache): 200 class SimpleCache(BaseCache):
81 """Simple memory cache for single process environments. This class exists 201 """Simple memory cache for single process environments. This class exists
82 mainly for the development server and is not 100% thread safe. It tries 202 mainly for the development server and is not 100% thread safe. It tries
83 to use as many atomic operations as possible and no locks for simplicity 203 to use as many atomic operations as possible and no locks for simplicity
84 but it could happen under heavy load that keys are added multiple times. 204 but it could happen under heavy load that keys are added multiple times.
205
206 :param threshold: the maximum number of items the cache stores before
207 it starts deleting some.
208 :param default_timeout: the default timeout that is used if no timeout is
209 specified on :meth:`~BaseCache.set`.
85 """ 210 """
86 211
87 def __init__(self, threshold=500, default_timeout=300): 212 def __init__(self, threshold=500, default_timeout=300):
88 BaseCache.__init__(self, default_timeout) 213 BaseCache.__init__(self, default_timeout)
89 self._cache = {} 214 self._cache = {}
124 _test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match 249 _test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match
125 250
126 class MemcachedCache(BaseCache): 251 class MemcachedCache(BaseCache):
127 """A cache that uses memcached as backend. 252 """A cache that uses memcached as backend.
128 253
254 The first argument can either be a list or tuple of server addresses
255 in which case Werkzeug tries to import the memcache module and connect
256 to it, or an object that resembles the API of a :class:`memcache.Client`.
257
129 Implementation notes: This cache backend works around some limitations in 258 Implementation notes: This cache backend works around some limitations in
130 memcached to simplify the interface. For example unicode keys are encoded 259 memcached to simplify the interface. For example unicode keys are encoded
131 to utf-8 on the fly. Methods such as `get_dict` return the keys in the 260 to utf-8 on the fly. Methods such as :meth:`~BaseCache.get_dict` return
132 same format as passed. Furthermore all get methods silently ignore key 261 the keys in the same format as passed. Furthermore all get methods
133 errors to not cause problems when untrusted user data is passed to the get 262 silently ignore key errors to not cause problems when untrusted user data
134 methods which is often the case in web applications. 263 is passed to the get methods which is often the case in web applications.
264
265 :param servers: a list or tuple of server addresses or alternatively
266 a :class:`memcache.Client` or a compatible client.
267 :param default_timeout: the default timeout that is used if no timeout is
268 specified on :meth:`~BaseCache.set`.
269 :param key_prefix: a prefix that is added before all keys. This makes it
270 possible to use the same memcached server for different
271 applications. Keep in mind that
272 :meth:`~BaseCache.clear` will also clear keys with a
273 different prefix.
135 """ 274 """
136 275
137 def __init__(self, servers, default_timeout=300): 276 def __init__(self, servers, default_timeout=300, key_prefix=None):
138 BaseCache.__init__(self, default_timeout) 277 BaseCache.__init__(self, default_timeout)
139 if not have_memcache: 278 if isinstance(servers, (list, tuple)):
140 raise RuntimeError('no memcache module found')
141
142 # cmemcache has a bug that debuglog is not defined for the
143 # client. Whenever pickle fails you get a weird AttributError.
144 if is_cmemcache:
145 self._client = memcache.Client(map(str, servers))
146 try: 279 try:
147 self._client.debuglog = lambda *a: None 280 import cmemcache as memcache
148 except: 281 is_cmemcache = True
149 pass 282 except ImportError:
283 try:
284 import memcache
285 is_cmemcache = False
286 except ImportError:
287 raise RuntimeError('no memcache module found')
288
289 # cmemcache has a bug that debuglog is not defined for the
290 # client. Whenever pickle fails you get a weird AttributError.
291 if is_cmemcache:
292 client = memcache.Client(map(str, servers))
293 try:
294 client.debuglog = lambda *a: None
295 except:
296 pass
297 else:
298 client = memcache.Client(servers, False, HIGHEST_PROTOCOL)
150 else: 299 else:
151 self._client = memcache.Client(servers, False, HIGHEST_PROTOCOL) 300 client = servers
301
302 self._client = client
303 self.key_prefix = key_prefix
152 304
153 def get(self, key): 305 def get(self, key):
154 if isinstance(key, unicode): 306 if isinstance(key, unicode):
155 key = key.encode('utf-8') 307 key = key.encode('utf-8')
308 if self.key_prefix:
309 key = self.key_prefix + key
156 # memcached doesn't support keys longer than that. Because often 310 # memcached doesn't support keys longer than that. Because often
157 # checks for so long keys can occour because it's tested from user 311 # checks for so long keys can occour because it's tested from user
158 # submitted data etc we fail silently for getting. 312 # submitted data etc we fail silently for getting.
159 if _test_memcached_key(key): 313 if _test_memcached_key(key):
160 return self._client.get(key) 314 return self._client.get(key)
166 if isinstance(key, unicode): 320 if isinstance(key, unicode):
167 encoded_key = key.encode('utf-8') 321 encoded_key = key.encode('utf-8')
168 have_encoded_keys = True 322 have_encoded_keys = True
169 else: 323 else:
170 encoded_key = key 324 encoded_key = key
325 if self.key_prefix:
326 encoded_key = self.key_prefix + encoded_key
171 if _test_memcached_key(key): 327 if _test_memcached_key(key):
172 key_mapping[encoded_key] = key 328 key_mapping[encoded_key] = key
173 # the keys call here is important because otherwise cmemcache 329 # the keys call here is important because otherwise cmemcache
174 # does ugly things. What exaclty I don't know, i think it does 330 # does ugly things. What exaclty I don't know, i think it does
175 # Py_DECREF but quite frankly i don't care. 331 # Py_DECREF but quite frankly i don't care.
176 d = rv = self._client.get_multi(key_mapping.keys()) 332 d = rv = self._client.get_multi(key_mapping.keys())
177 if have_encoded_keys: 333 if have_encoded_keys or self.key_prefix:
178 rv = {} 334 rv = {}
179 for key, value in d.iteritems(): 335 for key, value in d.iteritems():
180 rv[key_mapping[key]] = value 336 rv[key_mapping[key]] = value
181 if len(rv) < len(keys): 337 if len(rv) < len(keys):
182 for key in keys: 338 for key in keys:
187 def add(self, key, value, timeout=None): 343 def add(self, key, value, timeout=None):
188 if timeout is None: 344 if timeout is None:
189 timeout = self.default_timeout 345 timeout = self.default_timeout
190 if isinstance(key, unicode): 346 if isinstance(key, unicode):
191 key = key.encode('utf-8') 347 key = key.encode('utf-8')
348 if self.key_prefix:
349 key = self.key_prefix + key
192 self._client.add(key, value, timeout) 350 self._client.add(key, value, timeout)
193 351
194 def set(self, key, value, timeout=None): 352 def set(self, key, value, timeout=None):
195 if timeout is None: 353 if timeout is None:
196 timeout = self.default_timeout 354 timeout = self.default_timeout
197 if isinstance(key, unicode): 355 if isinstance(key, unicode):
198 key = key.encode('utf-8') 356 key = key.encode('utf-8')
357 if self.key_prefix:
358 key = self.key_prefix + key
199 self._client.set(key, value, timeout) 359 self._client.set(key, value, timeout)
200 360
201 def get_many(self, *keys): 361 def get_many(self, *keys):
202 d = self.get_dict(*keys) 362 d = self.get_dict(*keys)
203 return [d[key] for key in keys] 363 return [d[key] for key in keys]
207 timeout = self.default_timeout 367 timeout = self.default_timeout
208 new_mapping = {} 368 new_mapping = {}
209 for key, value in mapping.iteritems(): 369 for key, value in mapping.iteritems():
210 if isinstance(key, unicode): 370 if isinstance(key, unicode):
211 key = key.encode('utf-8') 371 key = key.encode('utf-8')
372 if self.key_prefix:
373 key = self.key_prefix + key
212 new_mapping[key] = value 374 new_mapping[key] = value
213 self._client.set_multi(new_mapping, timeout) 375 self._client.set_multi(new_mapping, timeout)
214 376
215 def delete(self, key): 377 def delete(self, key):
216 if isinstance(key, unicode): 378 if isinstance(key, unicode):
217 key = key.encode('utf-8') 379 key = key.encode('utf-8')
218 self._client.delete(key) 380 if self.key_prefix:
381 key = self.key_prefix + key
382 if _test_memcached_key(key):
383 self._client.delete(key)
219 384
220 def delete_many(self, *keys): 385 def delete_many(self, *keys):
221 keys = list(keys) 386 new_keys = []
222 for idx, key in enumerate(keys): 387 for key in keys:
223 if isinstance(key, unicode): 388 if isinstance(key, unicode):
224 keys[idx] = key.encode('utf-8') 389 key = key.encode('utf-8')
225 self._client.delete_multi(keys) 390 if self.key_prefix:
391 key = self.key_prefix + key
392 if _test_memcached_key(key):
393 new_keys.append(key)
394 self._client.delete_multi(new_keys)
226 395
227 def clear(self): 396 def clear(self):
228 self._client.flush_all() 397 self._client.flush_all()
229 398
230 def inc(self, key, delta=1): 399 def inc(self, key, delta=1):
231 if isinstance(key, unicode): 400 if isinstance(key, unicode):
232 key = key.encode('utf-8') 401 key = key.encode('utf-8')
233 self._client.incr(key, key, delta) 402 if self.key_prefix:
403 key = self.key_prefix + key
404 self._client.incr(key, delta)
234 405
235 def dec(self, key, delta=1): 406 def dec(self, key, delta=1):
236 if isinstance(key, unicode): 407 if isinstance(key, unicode):
237 key = key.encode('utf-8') 408 key = key.encode('utf-8')
238 self._client.decr(key, key, delta) 409 if self.key_prefix:
410 key = self.key_prefix + key
411 self._client.decr(key, delta)
412
413
414 class GAEMemcachedCache(MemcachedCache):
415 """Connects to the Google appengine memcached Cache.
416
417 :param default_timeout: the default timeout that is used if no timeout is
418 specified on :meth:`~BaseCache.set`.
419 :param key_prefix: a prefix that is added before all keys. This makes it
420 possible to use the same memcached server for different
421 applications. Keep in mind that
422 :meth:`~BaseCache.clear` will also clear keys with a
423 different prefix.
424 """
425
426 def __init__(self, default_timeout=300, key_prefix=None):
427 from google.appengine.api import memcache
428 MemcachedCache.__init__(self, memcache.Client(),
429 default_timeout, key_prefix)
239 430
240 431
241 class FileSystemCache(BaseCache): 432 class FileSystemCache(BaseCache):
242 """A cache that stores the items on the file system.""" 433 """A cache that stores the items on the file system. This cache depends
434 on being the only user of the `cache_dir`. Make absolutely sure that
435 nobody but this cache stores files there or otherwise the chace will
436 randomely delete files therein.
437
438 :param cache_dir: the directory where cached files are stored.
439 :param threshold: the maximum number of items the cache stores before
440 it starts deleting some.
441 :param default_timeout: the default timeout that is used if no timeout is
442 specified on :meth:`~BaseCache.set`.
443 """
243 444
244 def __init__(self, cache_dir, threshold=500, default_timeout=300): 445 def __init__(self, cache_dir, threshold=500, default_timeout=300):
245 BaseCache.__init__(self, default_timeout) 446 BaseCache.__init__(self, default_timeout)
246 self._path = cache_dir 447 self._path = cache_dir
247 self._threshold = threshold 448 self._threshold = threshold
253 if len(entries) > self._threshold: 454 if len(entries) > self._threshold:
254 now = time() 455 now = time()
255 for idx, key in enumerate(entries): 456 for idx, key in enumerate(entries):
256 try: 457 try:
257 f = file(self._get_filename(key)) 458 f = file(self._get_filename(key))
258 if pickle.load(f) > now and idx % 3 != 0: 459 if load(f) > now and idx % 3 != 0:
259 f.close() 460 f.close()
260 continue 461 continue
261 except: 462 except:
262 f.close() 463 f.close()
263 self.delete(key) 464 self.delete(key)