comparison MoinMoin/support/werkzeug/contrib/cache.py @ 6129:7f12cf241d5e

update werkzeug to 0.12.1, update CHANGES
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Tue, 11 Apr 2017 22:42:23 +0200
parents 9f12f41504fc
children
comparison
equal deleted inserted replaced
6128:561b7a9c2bd9 6129:7f12cf241d5e
21 something else. When the request comes in you check if the current page 21 something else. When the request comes in you check if the current page
22 is already in the cache and if so, you're returning it from the cache. 22 is already in the cache and if so, you're returning it from the cache.
23 Otherwise you generate the page and put it into the cache. (Or a fragment 23 Otherwise you generate the page and put it into the cache. (Or a fragment
24 of the page, you don't have to cache the full thing) 24 of the page, you don't have to cache the full thing)
25 25
26 Here is a simple example of how to cache a sidebar for a template:: 26 Here is a simple example of how to cache a sidebar for 5 minutes::
27 27
28 def get_sidebar(user): 28 def get_sidebar(user):
29 identifier = 'sidebar_for/user%d' % user.id 29 identifier = 'sidebar_for/user%d' % user.id
30 value = cache.get(identifier) 30 value = cache.get(identifier)
31 if value is not None: 31 if value is not None:
58 """ 58 """
59 import os 59 import os
60 import re 60 import re
61 import errno 61 import errno
62 import tempfile 62 import tempfile
63 import platform
63 from hashlib import md5 64 from hashlib import md5
64 from time import time 65 from time import time
65 try: 66 try:
66 import cPickle as pickle 67 import cPickle as pickle
67 except ImportError: # pragma: no cover 68 except ImportError: # pragma: no cover
91 class BaseCache(object): 92 class BaseCache(object):
92 93
93 """Baseclass for the cache systems. All the cache systems implement this 94 """Baseclass for the cache systems. All the cache systems implement this
94 API or a superset of it. 95 API or a superset of it.
95 96
96 :param default_timeout: the default timeout (in seconds) that is used if no 97 :param default_timeout: the default timeout (in seconds) that is used if
97 timeout is specified on :meth:`set`. A timeout of 0 98 no timeout is specified on :meth:`set`. A timeout
98 indicates that the cache never expires. 99 of 0 indicates that the cache never expires.
99 """ 100 """
100 101
101 def __init__(self, default_timeout=300): 102 def __init__(self, default_timeout=300):
102 self.default_timeout = default_timeout 103 self.default_timeout = default_timeout
104
105 def _normalize_timeout(self, timeout):
106 if timeout is None:
107 timeout = self.default_timeout
108 return timeout
103 109
104 def get(self, key): 110 def get(self, key):
105 """Look up key in the cache and return the value for it. 111 """Look up key in the cache and return the value for it.
106 112
107 :param key: the key to be looked up. 113 :param key: the key to be looked up.
147 """Add a new key/value to the cache (overwrites value, if key already 153 """Add a new key/value to the cache (overwrites value, if key already
148 exists in the cache). 154 exists in the cache).
149 155
150 :param key: the key to set 156 :param key: the key to set
151 :param value: the value for the key 157 :param value: the value for the key
152 :param timeout: the cache timeout for the key (if not specified, 158 :param timeout: the cache timeout for the key in seconds (if not
153 it uses the default timeout). A timeout of 0 idicates 159 specified, it uses the default timeout). A timeout of
154 that the cache never expires. 160 0 idicates that the cache never expires.
155 :returns: ``True`` if key has been updated, ``False`` for backend 161 :returns: ``True`` if key has been updated, ``False`` for backend
156 errors. Pickling errors, however, will raise a subclass of 162 errors. Pickling errors, however, will raise a subclass of
157 ``pickle.PickleError``. 163 ``pickle.PickleError``.
158 :rtype: boolean 164 :rtype: boolean
159 """ 165 """
163 """Works like :meth:`set` but does not overwrite the values of already 169 """Works like :meth:`set` but does not overwrite the values of already
164 existing keys. 170 existing keys.
165 171
166 :param key: the key to set 172 :param key: the key to set
167 :param value: the value for the key 173 :param value: the value for the key
168 :param timeout: the cache timeout for the key or the default 174 :param timeout: the cache timeout for the key in seconds (if not
169 timeout if not specified. A timeout of 0 indicates 175 specified, it uses the default timeout). A timeout of
170 that the cache never expires. 176 0 idicates that the cache never expires.
171 :returns: Same as :meth:`set`, but also ``False`` for already 177 :returns: Same as :meth:`set`, but also ``False`` for already
172 existing keys. 178 existing keys.
173 :rtype: boolean 179 :rtype: boolean
174 """ 180 """
175 return True 181 return True
176 182
177 def set_many(self, mapping, timeout=None): 183 def set_many(self, mapping, timeout=None):
178 """Sets multiple keys and values from a mapping. 184 """Sets multiple keys and values from a mapping.
179 185
180 :param mapping: a mapping with the keys/values to set. 186 :param mapping: a mapping with the keys/values to set.
181 :param timeout: the cache timeout for the key (if not specified, 187 :param timeout: the cache timeout for the key in seconds (if not
182 it uses the default timeout). A timeout of 0 188 specified, it uses the default timeout). A timeout of
183 indicates tht the cache never expires. 189 0 idicates that the cache never expires.
184 :returns: Whether all given keys have been set. 190 :returns: Whether all given keys have been set.
185 :rtype: boolean 191 :rtype: boolean
186 """ 192 """
187 rv = True 193 rv = True
188 for key, value in _items(mapping): 194 for key, value in _items(mapping):
216 ) 222 )
217 223
218 def clear(self): 224 def clear(self):
219 """Clears the cache. Keep in mind that not all caches support 225 """Clears the cache. Keep in mind that not all caches support
220 completely clearing the cache. 226 completely clearing the cache.
227
221 :returns: Whether the cache has been cleared. 228 :returns: Whether the cache has been cleared.
222 :rtype: boolean 229 :rtype: boolean
223 """ 230 """
224 return True 231 return True
225 232
287 if (expires != 0 and expires <= now) or idx % 3 == 0: 294 if (expires != 0 and expires <= now) or idx % 3 == 0:
288 toremove.append(key) 295 toremove.append(key)
289 for key in toremove: 296 for key in toremove:
290 self._cache.pop(key, None) 297 self._cache.pop(key, None)
291 298
292 def _get_expiration(self, timeout): 299 def _normalize_timeout(self, timeout):
293 if timeout is None: 300 timeout = BaseCache._normalize_timeout(self, timeout)
294 timeout = self.default_timeout
295 if timeout > 0: 301 if timeout > 0:
296 timeout = time() + timeout 302 timeout = time() + timeout
297 return timeout 303 return timeout
298 304
299 def get(self, key): 305 def get(self, key):
303 return pickle.loads(value) 309 return pickle.loads(value)
304 except (KeyError, pickle.PickleError): 310 except (KeyError, pickle.PickleError):
305 return None 311 return None
306 312
307 def set(self, key, value, timeout=None): 313 def set(self, key, value, timeout=None):
308 expires = self._get_expiration(timeout) 314 expires = self._normalize_timeout(timeout)
309 self._prune() 315 self._prune()
310 self._cache[key] = (expires, pickle.dumps(value, 316 self._cache[key] = (expires, pickle.dumps(value,
311 pickle.HIGHEST_PROTOCOL)) 317 pickle.HIGHEST_PROTOCOL))
312 return True 318 return True
313 319
314 def add(self, key, value, timeout=None): 320 def add(self, key, value, timeout=None):
315 expires = self._get_expiration(timeout) 321 expires = self._normalize_timeout(timeout)
316 self._prune() 322 self._prune()
317 item = (expires, pickle.dumps(value, 323 item = (expires, pickle.dumps(value,
318 pickle.HIGHEST_PROTOCOL)) 324 pickle.HIGHEST_PROTOCOL))
319 if key in self._cache: 325 if key in self._cache:
320 return False 326 return False
389 if self.key_prefix: 395 if self.key_prefix:
390 key = self.key_prefix + key 396 key = self.key_prefix + key
391 return key 397 return key
392 398
393 def _normalize_timeout(self, timeout): 399 def _normalize_timeout(self, timeout):
394 if timeout is None: 400 timeout = BaseCache._normalize_timeout(self, timeout)
395 timeout = self.default_timeout
396 if timeout > 0: 401 if timeout > 0:
397 timeout = int(time()) + timeout 402 timeout = int(time()) + timeout
398 return timeout 403 return timeout
399 404
400 def get(self, key): 405 def get(self, key):
559 db=db, **kwargs) 564 db=db, **kwargs)
560 else: 565 else:
561 self._client = host 566 self._client = host
562 self.key_prefix = key_prefix or '' 567 self.key_prefix = key_prefix or ''
563 568
564 def _get_expiration(self, timeout): 569 def _normalize_timeout(self, timeout):
565 if timeout is None: 570 timeout = BaseCache._normalize_timeout(self, timeout)
566 timeout = self.default_timeout
567 if timeout == 0: 571 if timeout == 0:
568 timeout = -1 572 timeout = -1
569 return timeout 573 return timeout
570 574
571 def dump_object(self, value): 575 def dump_object(self, value):
601 if self.key_prefix: 605 if self.key_prefix:
602 keys = [self.key_prefix + key for key in keys] 606 keys = [self.key_prefix + key for key in keys]
603 return [self.load_object(x) for x in self._client.mget(keys)] 607 return [self.load_object(x) for x in self._client.mget(keys)]
604 608
605 def set(self, key, value, timeout=None): 609 def set(self, key, value, timeout=None):
606 timeout = self._get_expiration(timeout) 610 timeout = self._normalize_timeout(timeout)
607 dump = self.dump_object(value) 611 dump = self.dump_object(value)
608 if timeout == -1: 612 if timeout == -1:
609 result = self._client.set(name=self.key_prefix + key, 613 result = self._client.set(name=self.key_prefix + key,
610 value=dump) 614 value=dump)
611 else: 615 else:
612 result = self._client.setex(name=self.key_prefix + key, 616 result = self._client.setex(name=self.key_prefix + key,
613 value=dump, time=timeout) 617 value=dump, time=timeout)
614 return result 618 return result
615 619
616 def add(self, key, value, timeout=None): 620 def add(self, key, value, timeout=None):
617 timeout = self._get_expiration(timeout) 621 timeout = self._normalize_timeout(timeout)
618 dump = self.dump_object(value) 622 dump = self.dump_object(value)
619 return ( 623 return (
620 self._client.setnx(name=self.key_prefix + key, value=dump) and 624 self._client.setnx(name=self.key_prefix + key, value=dump) and
621 self._client.expire(name=self.key_prefix + key, time=timeout) 625 self._client.expire(name=self.key_prefix + key, time=timeout)
622 ) 626 )
623 627
624 def set_many(self, mapping, timeout=None): 628 def set_many(self, mapping, timeout=None):
625 timeout = self._get_expiration(timeout) 629 timeout = self._normalize_timeout(timeout)
626 # Use transaction=False to batch without calling redis MULTI 630 # Use transaction=False to batch without calling redis MULTI
627 # which is not supported by twemproxy 631 # which is not supported by twemproxy
628 pipe = self._client.pipeline(transaction=False) 632 pipe = self._client.pipeline(transaction=False)
629 633
630 for key, value in _items(mapping): 634 for key, value in _items(mapping):
696 os.makedirs(self._path) 700 os.makedirs(self._path)
697 except OSError as ex: 701 except OSError as ex:
698 if ex.errno != errno.EEXIST: 702 if ex.errno != errno.EEXIST:
699 raise 703 raise
700 704
705 def _normalize_timeout(self, timeout):
706 timeout = BaseCache._normalize_timeout(self, timeout)
707 if timeout != 0:
708 timeout = time() + timeout
709 return int(timeout)
710
701 def _list_dir(self): 711 def _list_dir(self):
702 """return a list of (fully qualified) cache filenames 712 """return a list of (fully qualified) cache filenames
703 """ 713 """
704 return [os.path.join(self._path, fn) for fn in os.listdir(self._path) 714 return [os.path.join(self._path, fn) for fn in os.listdir(self._path)
705 if not fn.endswith(self._fs_transaction_suffix)] 715 if not fn.endswith(self._fs_transaction_suffix)]
706 716
707 def _prune(self): 717 def _prune(self):
708 entries = self._list_dir() 718 entries = self._list_dir()
709 if len(entries) > self._threshold: 719 if len(entries) > self._threshold:
710 now = time() 720 now = time()
711 try: 721 for idx, fname in enumerate(entries):
712 for idx, fname in enumerate(entries): 722 try:
713 remove = False 723 remove = False
714 with open(fname, 'rb') as f: 724 with open(fname, 'rb') as f:
715 expires = pickle.load(f) 725 expires = pickle.load(f)
716 remove = (expires != 0 and expires <= now) or idx % 3 == 0 726 remove = (expires != 0 and expires <= now) or idx % 3 == 0
717 727
718 if remove: 728 if remove:
719 os.remove(fname) 729 os.remove(fname)
720 except (IOError, OSError): 730 except (IOError, OSError):
721 pass 731 pass
722 732
723 def clear(self): 733 def clear(self):
724 for fname in self._list_dir(): 734 for fname in self._list_dir():
725 try: 735 try:
726 os.remove(fname) 736 os.remove(fname)
752 if not os.path.exists(filename): 762 if not os.path.exists(filename):
753 return self.set(key, value, timeout) 763 return self.set(key, value, timeout)
754 return False 764 return False
755 765
756 def set(self, key, value, timeout=None): 766 def set(self, key, value, timeout=None):
757 if timeout is None: 767 timeout = self._normalize_timeout(timeout)
758 timeout = int(time() + self.default_timeout)
759 elif timeout != 0:
760 timeout = int(time() + timeout)
761 filename = self._get_filename(key) 768 filename = self._get_filename(key)
762 self._prune() 769 self._prune()
763 try: 770 try:
764 fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix, 771 fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix,
765 dir=self._path) 772 dir=self._path)
791 else: 798 else:
792 os.remove(filename) 799 os.remove(filename)
793 return False 800 return False
794 except (IOError, OSError, pickle.PickleError): 801 except (IOError, OSError, pickle.PickleError):
795 return False 802 return False
803
804
805 class UWSGICache(BaseCache):
806 """ Implements the cache using uWSGI's caching framework.
807
808 .. note::
809 This class cannot be used when running under PyPy, because the uWSGI
810 API implementation for PyPy is lacking the needed functionality.
811
812 :param default_timeout: The default timeout in seconds.
813 :param cache: The name of the caching instance to connect to, for
814 example: mycache@localhost:3031, defaults to an empty string, which
815 means uWSGI will cache in the local instance. If the cache is in the
816 same instance as the werkzeug app, you only have to provide the name of
817 the cache.
818 """
819 def __init__(self, default_timeout=300, cache=''):
820 BaseCache.__init__(self, default_timeout)
821
822 if platform.python_implementation() == 'PyPy':
823 raise RuntimeError("uWSGI caching does not work under PyPy, see "
824 "the docs for more details.")
825
826 try:
827 import uwsgi
828 self._uwsgi = uwsgi
829 except ImportError:
830 raise RuntimeError("uWSGI could not be imported, are you "
831 "running under uWSGI?")
832
833 self.cache = cache
834
835 def get(self, key):
836 rv = self._uwsgi.cache_get(key, self.cache)
837 if rv is None:
838 return
839 return pickle.loads(rv)
840
841 def delete(self, key):
842 return self._uwsgi.cache_del(key, self.cache)
843
844 def set(self, key, value, timeout=None):
845 return self._uwsgi.cache_update(key, pickle.dumps(value),
846 self._normalize_timeout(timeout),
847 self.cache)
848
849 def add(self, key, value, timeout=None):
850 return self._uwsgi.cache_set(key, pickle.dumps(value),
851 self._normalize_timeout(timeout),
852 self.cache)
853
854 def clear(self):
855 return self._uwsgi.cache_clear(self.cache)
856
857 def has(self, key):
858 return self._uwsgi.cache_exists(key, self.cache) is not None