diff MoinMoin/support/werkzeug/contrib/cache.py @ 5801:8de563c487be

upgrade werkzeug to 0.8.1, document current bundled version and current minimum requirement (0.6, for py 2.7 compatibility)
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Thu, 01 Dec 2011 01:34:45 +0100
parents 246ba4eecab2
children fc1f97a47c7e
line wrap: on
line diff
--- a/MoinMoin/support/werkzeug/contrib/cache.py	Sat Mar 12 23:48:11 2011 +0100
+++ b/MoinMoin/support/werkzeug/contrib/cache.py	Thu Dec 01 01:34:45 2011 +0100
@@ -19,11 +19,11 @@
     Caching is pretty simple.  Basically you have a cache object lurking around
     somewhere that is connected to a remote cache or the file system or
     something else.  When the request comes in you check if the current page
-    is already in the cache and if, you're returning it.  Otherwise you generate
-    the page and put it into the cache.  (Or a fragment of the page, you don't
-    have to cache the full thing)
+    is already in the cache and if so, you're returning it from the cache.
+    Otherwise you generate the page and put it into the cache. (Or a fragment
+    of the page, you don't have to cache the full thing)
 
-    Here a simple example of how to cache a sidebar for a template::
+    Here is a simple example of how to cache a sidebar for a template::
 
         def get_sidebar(user):
             identifier = 'sidebar_for/user%d' % user.id
@@ -38,7 +38,7 @@
     =======================
 
     To create a cache object you just import the cache system of your choice
-    from the cache module and instanciate it.  Then you can start working
+    from the cache module and instantiate it.  Then you can start working
     with that object:
 
     >>> from werkzeug.contrib.cache import SimpleCache
@@ -50,21 +50,42 @@
     True
 
     Please keep in mind that you have to create the cache and put it somewhere
-    you have access to it (either as a module global you can import or if you
-    put it onto your WSGI application).
+    you have access to it (either as a module global you can import or you just
+    put it into your WSGI application).
 
-    :copyright: (c) 2009 by the Werkzeug Team, see AUTHORS for more details.
+    :copyright: (c) 2011 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
 import os
 import re
+import tempfile
 try:
     from hashlib import md5
 except ImportError:
     from md5 import new as md5
 from itertools import izip
 from time import time
-from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL
+from werkzeug.posixemulation import rename
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+
+def _items(mappingorseq):
+    """Wrapper for efficient iteration over mappings represented by dicts
+    or sequences::
+
+        >>> for k, v in _items((i, i*i) for i in xrange(5)):
+        ...    assert k*k == v
+
+        >>> for k, v in _items(dict((i, i*i) for i in xrange(5))):
+        ...    assert k*k == v
+
+    """
+    return mappingorseq.iteritems() if hasattr(mappingorseq, 'iteritems') \
+        else mappingorseq
 
 
 class BaseCache(object):
@@ -79,8 +100,8 @@
         self.default_timeout = default_timeout
 
     def get(self, key):
-        """Looks up key in the cache and returns it.  If the key does not
-        exist `None` is returned instead.
+        """Looks up key in the cache and returns the value for it.
+        If the key does not exist `None` is returned instead.
 
         :param key: the key to be looked up.
         """
@@ -95,8 +116,8 @@
         pass
 
     def get_many(self, *keys):
-        """Returns a list of keys.  For each key a item in the list is
-        created.  Example::
+        """Returns a list of values for the given keys.
+        For each key a item in the list is created.  Example::
 
             foo, bar = cache.get_many("foo", "bar")
 
@@ -121,18 +142,19 @@
         return dict(izip(keys, self.get_many(*keys)))
 
     def set(self, key, value, timeout=None):
-        """Adds or overrides a key in the cache.
+        """Adds a new key/value to the cache (overwrites value, if key already
+        exists in the cache).
 
         :param key: the key to set
         :param value: the value for the key
-        :param timeout: the cache timeout for the key or the default
-                        timeout if not specified.
+        :param timeout: the cache timeout for the key (if not specified,
+                        it uses the default timeout).
         """
         pass
 
     def add(self, key, value, timeout=None):
-        """Works like :meth:`set` but does not override already existing
-        values.
+        """Works like :meth:`set` but does not overwrite the values of already
+        existing keys.
 
         :param key: the key to set
         :param value: the value for the key
@@ -142,13 +164,13 @@
         pass
 
     def set_many(self, mapping, timeout=None):
-        """Sets multiple keys and values from a dict.
+        """Sets multiple keys and values from a mapping.
 
-        :param mapping: a dict with the values to set.
-        :param timeout: the cache timeout for the key or the default
-                        timeout if not specified.
+        :param mapping: a mapping with the keys/values to set.
+        :param timeout: the cache timeout for the key (if not specified,
+                        it uses the default timeout).
         """
-        for key, value in mapping.iteritems():
+        for key, value in _items(mapping):
             self.set(key, value, timeout)
 
     def delete_many(self, *keys):
@@ -162,7 +184,7 @@
 
     def clear(self):
         """Clears the cache.  Keep in mind that not all caches support
-        clearning of the full cache.
+        completely clearing the cache.
         """
         pass
 
@@ -226,20 +248,22 @@
         now = time()
         expires, value = self._cache.get(key, (0, None))
         if expires > time():
-            return loads(value)
+            return pickle.loads(value)
 
     def set(self, key, value, timeout=None):
         if timeout is None:
             timeout = self.default_timeout
         self._prune()
-        self._cache[key] = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
+        self._cache[key] = (time() + timeout, pickle.dumps(value,
+            pickle.HIGHEST_PROTOCOL))
 
     def add(self, key, value, timeout=None):
         if timeout is None:
             timeout = self.default_timeout
         if len(self._cache) > self._threshold:
             self._prune()
-        item = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
+        item = (time() + timeout, pickle.dumps(value,
+            pickle.HIGHEST_PROTOCOL))
         self._cache.setdefault(key, item)
 
     def delete(self, key):
@@ -251,9 +275,10 @@
 class MemcachedCache(BaseCache):
     """A cache that uses memcached as backend.
 
-    The first argument can either be a list or tuple of server addresses
-    in which case Werkzeug tries to import the memcache module and connect
-    to it, or an object that resembles the API of a :class:`memcache.Client`.
+    The first argument can either be an object that resembles the API of a
+    :class:`memcache.Client` or a tuple/list of server addresses. In the
+    event that a tuple/list is passed, Werkzeug tries to import the best
+    available memcache library.
 
     Implementation notes:  This cache backend works around some limitations in
     memcached to simplify the interface.  For example unicode keys are encoded
@@ -273,33 +298,19 @@
                        different prefix.
     """
 
-    def __init__(self, servers, default_timeout=300, key_prefix=None):
+    def __init__(self, servers=None, default_timeout=300, key_prefix=None):
         BaseCache.__init__(self, default_timeout)
-        if isinstance(servers, (list, tuple)):
-            try:
-                import cmemcache as memcache
-                is_cmemcache = True
-            except ImportError:
-                try:
-                    import memcache
-                    is_cmemcache = False
-                except ImportError:
-                    raise RuntimeError('no memcache module found')
+        if servers is None or isinstance(servers, (list, tuple)):
+            if servers is None:
+                servers = ['127.0.0.1:11211']
+            self._client = self.import_preferred_memcache_lib(servers)
+            if self._client is None:
+                raise RuntimeError('no memcache module found')
+        else:
+            # NOTE: servers is actually an already initialized memcache
+            # client.
+            self._client = servers
 
-            # cmemcache has a bug that debuglog is not defined for the
-            # client.  Whenever pickle fails you get a weird AttributError.
-            if is_cmemcache:
-                client = memcache.Client(map(str, servers))
-                try:
-                    client.debuglog = lambda *a: None
-                except:
-                    pass
-            else:
-                client = memcache.Client(servers, False, HIGHEST_PROTOCOL)
-        else:
-            client = servers
-
-        self._client = client
         self.key_prefix = key_prefix
 
     def get(self, key):
@@ -316,7 +327,7 @@
     def get_dict(self, *keys):
         key_mapping = {}
         have_encoded_keys = False
-        for idx, key in enumerate(keys):
+        for key in keys:
             if isinstance(key, unicode):
                 encoded_key = key.encode('utf-8')
                 have_encoded_keys = True
@@ -326,9 +337,6 @@
                 encoded_key = self.key_prefix + encoded_key
             if _test_memcached_key(key):
                 key_mapping[encoded_key] = key
-        # the keys call here is important because otherwise cmemcache
-        # does ugly things.  What exaclty I don't know, i think it does
-        # Py_DECREF but quite frankly i don't care.
         d = rv = self._client.get_multi(key_mapping.keys())
         if have_encoded_keys or self.key_prefix:
             rv = {}
@@ -366,7 +374,7 @@
         if timeout is None:
             timeout = self.default_timeout
         new_mapping = {}
-        for key, value in mapping.iteritems():
+        for key, value in _items(mapping):
             if isinstance(key, unicode):
                 key = key.encode('utf-8')
             if self.key_prefix:
@@ -410,58 +418,210 @@
             key = self.key_prefix + key
         self._client.decr(key, delta)
 
+    def import_preferred_memcache_lib(self, servers):
+        """Returns an initialized memcache client.  Used by the constructor."""
+        try:
+            import pylibmc
+        except ImportError:
+            pass
+        else:
+            return pylibmc.Client(servers)
 
-class GAEMemcachedCache(MemcachedCache):
-    """Connects to the Google appengine memcached Cache.
+        try:
+            from google.appengine.api import memcache
+        except ImportError:
+            pass
+        else:
+            return memcache.Client()
 
+        try:
+            import memcache
+        except ImportError:
+            pass
+        else:
+            return memcache.Client(servers)
+
+
+# backwards compatibility
+GAEMemcachedCache = MemcachedCache
+
+
+class RedisCache(BaseCache):
+    """Uses the Redis key-value store as a cache backend.
+
+    The first argument can be either a string denoting address of the Redis
+    server or an object resembling an instance of a redis.Redis class.
+
+    Note: Python Redis API already takes care of encoding unicode strings on
+    the fly.
+
+    .. versionadded:: 0.7
+
+    .. versionadded:: 0.8
+       `key_prefix` was added.
+
+    .. versionchanged:: 0.8
+       This cache backend now properly serializes objects.
+
+    :param host: address of the Redis server or an object which API is
+                 compatible with the official Python Redis client (redis-py).
+    :param port: port number on which Redis server listens for connections
     :param default_timeout: the default timeout that is used if no timeout is
                             specified on :meth:`~BaseCache.set`.
-    :param key_prefix: a prefix that is added before all keys.  This makes it
-                       possible to use the same memcached server for different
-                       applications.  Keep in mind that
-                       :meth:`~BaseCache.clear` will also clear keys with a
-                       different prefix.
+    :param key_prefix: A prefix that should be added to all keys.
     """
 
-    def __init__(self, default_timeout=300, key_prefix=None):
-        from google.appengine.api import memcache
-        MemcachedCache.__init__(self, memcache.Client(),
-                                default_timeout, key_prefix)
+    def __init__(self, host='localhost', port=6379, default_timeout=300,
+                 key_prefix=None):
+        BaseCache.__init__(self, default_timeout)
+        if isinstance(host, basestring):
+            try:
+                import redis
+            except ImportError:
+                raise RuntimeError('no redis module found')
+            self._client = redis.Redis(host=host, port=port)
+        else:
+            self._client = host
+        self.key_prefix = key_prefix or ''
+
+    def dump_object(self, value):
+        """Dumps an object into a string for redis.  By default it serializes
+        integers as regular string and pickle dumps everything else.
+        """
+        if isinstance(value, (int, long)):
+            return str(value)
+        return '!' + pickle.dumps(value)
+
+    def load_object(self, value):
+        """The reversal of :meth:`dump_object`.  This might be callde with
+        None.
+        """
+        if value is None:
+            return None
+        if value.startswith('!'):
+            return pickle.loads(value[1:])
+        try:
+            return int(value)
+        except ValueError:
+            # before 0.8 we did not have serialization.  Still support that.
+            return value
+
+    def get(self, key):
+        return self.load_object(self._client.get(self.key_prefix + key))
+
+    def get_many(self, *keys):
+        if self.key_prefix:
+            keys = [self.key_prefix + key for key in keys]
+        return [self.load_object(x) for x in self._client.mget(keys)]
+
+    def set(self, key, value, timeout=None):
+        if timeout is None:
+            timeout = self.default_timeout
+        dump = self.dump_object(value)
+        self._client.setex(self.key_prefix + key, dump, timeout)
+
+    def add(self, key, value, timeout=None):
+        if timeout is None:
+            timeout = self.default_timeout
+        dump = self.dump_object(value)
+        added = self._client.setnx(self.key_prefix + key, dump)
+        if added:
+            self._client.expire(self.key_prefix + key, timeout)
+
+    def set_many(self, mapping, timeout=None):
+        if timeout is None:
+            timeout = self.default_timeout
+        pipe = self._client.pipeline()
+        for key, value in _items(mapping):
+            dump = self.dump_object(value)
+            pipe.setex(self.key_prefix + key, dump, timeout)
+        pipe.execute()
+
+    def delete(self, key):
+        self._client.delete(self.key_prefix + key)
+
+    def delete_many(self, *keys):
+        if not keys:
+            return
+        if self.key_prefix:
+            keys = [self.key_prefix + key for key in keys]
+        self._client.delete(*keys)
+
+    def clear(self):
+        if self.key_prefix:
+            keys = self._client.keys(self.key_prefix + '*')
+            if keys:
+                self._client.delete(*keys)
+        else:
+            self._client.flushdb()
+
+    def inc(self, key, delta=1):
+        return self._client.incr(self.key_prefix + key, delta)
+
+    def dec(self, key, delta=1):
+        return self._client.decr(self.key_prefix + key, delta)
 
 
 class FileSystemCache(BaseCache):
     """A cache that stores the items on the file system.  This cache depends
     on being the only user of the `cache_dir`.  Make absolutely sure that
-    nobody but this cache stores files there or otherwise the chace will
-    randomely delete files therein.
+    nobody but this cache stores files there or otherwise the cache will
+    randomly delete files therein.
 
-    :param cache_dir: the directory where cached files are stored.
+    :param cache_dir: the directory where cache files are stored.
     :param threshold: the maximum number of items the cache stores before
                       it starts deleting some.
     :param default_timeout: the default timeout that is used if no timeout is
                             specified on :meth:`~BaseCache.set`.
+    :param mode: the file mode wanted for the cache files, default 0600
     """
 
-    def __init__(self, cache_dir, threshold=500, default_timeout=300):
+    #: used for temporary files by the FileSystemCache
+    _fs_transaction_suffix = '.__wz_cache'
+
+    def __init__(self, cache_dir, threshold=500, default_timeout=300, mode=0600):
         BaseCache.__init__(self, default_timeout)
         self._path = cache_dir
         self._threshold = threshold
+        self._mode = mode
         if not os.path.exists(self._path):
             os.makedirs(self._path)
 
+    def _list_dir(self):
+        """return a list of (fully qualified) cache filenames
+        """
+        return [os.path.join(self._path, fn) for fn in os.listdir(self._path)
+                if not fn.endswith(self._fs_transaction_suffix)]
+
     def _prune(self):
-        entries = os.listdir(self._path)
+        entries = self._list_dir()
         if len(entries) > self._threshold:
             now = time()
-            for idx, key in enumerate(entries):
+            for idx, fname in enumerate(entries):
+                remove = False
+                f = None
                 try:
-                    f = file(self._get_filename(key))
-                    if load(f) > now and idx % 3 != 0:
-                        f.close()
-                        continue
-                except:
-                    f.close()
-                self.delete(key)
+                    try:
+                        f = open(fname, 'rb')
+                        expires = pickle.load(f)
+                        remove = expires <= now or idx % 3 == 0
+                    finally:
+                        if f is not None:
+                            f.close()
+                except Exception:
+                    pass
+                if remove:
+                    try:
+                        os.remove(fname)
+                    except (IOError, OSError):
+                        pass
+
+    def clear(self):
+        for fname in self._list_dir():
+            try:
+                os.remove(fname)
+            except (IOError, OSError):
+                pass
 
     def _get_filename(self, key):
         hash = md5(key).hexdigest()
@@ -470,14 +630,14 @@
     def get(self, key):
         filename = self._get_filename(key)
         try:
-            f = file(filename, 'rb')
+            f = open(filename, 'rb')
             try:
-                if load(f) >= time():
-                    return load(f)
+                if pickle.load(f) >= time():
+                    return pickle.load(f)
             finally:
                 f.close()
             os.remove(filename)
-        except:
+        except Exception:
             return None
 
     def add(self, key, value, timeout=None):
@@ -491,12 +651,16 @@
         filename = self._get_filename(key)
         self._prune()
         try:
-            f = file(filename, 'wb')
+            fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix,
+                                       dir=self._path)
+            f = os.fdopen(fd, 'wb')
             try:
-                dump(int(time() + timeout), f, 1)
-                dump(value, f, HIGHEST_PROTOCOL)
+                pickle.dump(int(time() + timeout), f, 1)
+                pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
             finally:
                 f.close()
+            rename(tmp, filename)
+            os.chmod(filename, self._mode)
         except (IOError, OSError):
             pass
 
@@ -505,7 +669,3 @@
             os.remove(self._get_filename(key))
         except (IOError, OSError):
             pass
-
-    def clear(self):
-        for key in os.listdir(self._path):
-            self.delete(key)