diff MoinMoin/support/werkzeug/contrib/cache.py @ 4609:246ba4eecab2

updated werkzeug to 0.5.pre20090228
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sat, 28 Feb 2009 00:08:31 +0100
parents c689dfa55de1
children 8de563c487be
line wrap: on
line diff
--- a/MoinMoin/support/werkzeug/contrib/cache.py	Fri Feb 27 23:30:37 2009 +0100
+++ b/MoinMoin/support/werkzeug/contrib/cache.py	Sat Feb 28 00:08:31 2009 +0100
@@ -3,14 +3,57 @@
     werkzeug.contrib.cache
     ~~~~~~~~~~~~~~~~~~~~~~
 
-    Small helper module that provides a simple interface to memcached, a
-    simple django-inspired in-process cache and a file system based cache.
+    The main problem with dynamic Web sites is, well, they're dynamic.  Each
+    time a user requests a page, the webserver executes a lot of code, queries
+    the database, renders templates until the visitor gets the page he sees.
 
-    The idea is that it's possible to switch caching systems without changing
-    much code in the application.
+    This is a lot more expensive than just loading a file from the file system
+    and sending it to the visitor.
 
+    For most Web applications, this overhead isn't a big deal but once it
+    becomes, you will be glad to have a cache system in place.
 
-    :copyright: 2007-2008 by Armin Ronacher.
+    How Caching Works
+    =================
+
+    Caching is pretty simple.  Basically you have a cache object lurking around
+    somewhere that is connected to a remote cache or the file system or
+    something else.  When the request comes in you check if the current page
+    is already in the cache and if, you're returning it.  Otherwise you generate
+    the page and put it into the cache.  (Or a fragment of the page, you don't
+    have to cache the full thing)
+
+    Here a simple example of how to cache a sidebar for a template::
+
+        def get_sidebar(user):
+            identifier = 'sidebar_for/user%d' % user.id
+            value = cache.get(identifier)
+            if value is not None:
+                return value
+            value = generate_sidebar_for(user=user)
+            cache.set(identifier, value, timeout=60 * 5)
+            return value
+
+    Creating a Cache Object
+    =======================
+
+    To create a cache object you just import the cache system of your choice
+    from the cache module and instanciate it.  Then you can start working
+    with that object:
+
+    >>> from werkzeug.contrib.cache import SimpleCache
+    >>> c = SimpleCache()
+    >>> c.set("foo", "value")
+    >>> c.get("foo")
+    'value'
+    >>> c.get("missing") is None
+    True
+
+    Please keep in mind that you have to create the cache and put it somewhere
+    you have access to it (either as a module global you can import or if you
+    put it onto your WSGI application).
+
+    :copyright: (c) 2009 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
 import os
@@ -23,58 +66,135 @@
 from time import time
 from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL
 
-have_memcache = True
-try:
-    import cmemcache as memcache
-    is_cmemcache = True
-except ImportError:
-    try:
-        import memcache
-        is_cmemcache = False
-    except ImportError:
-        have_memcache = False
-
 
 class BaseCache(object):
-    """Baseclass for the cache systems."""
+    """Baseclass for the cache systems.  All the cache systems implement this
+    API or a superset of it.
+
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`set`.
+    """
 
     def __init__(self, default_timeout=300):
         self.default_timeout = default_timeout
 
     def get(self, key):
+        """Looks up key in the cache and returns it.  If the key does not
+        exist `None` is returned instead.
+
+        :param key: the key to be looked up.
+        """
         return None
-    delete = get
+
+    def delete(self, key):
+        """Deletes `key` from the cache.  If it does not exist in the cache
+        nothing happens.
+
+        :param key: the key to delete.
+        """
+        pass
 
     def get_many(self, *keys):
+        """Returns a list of keys.  For each key a item in the list is
+        created.  Example::
+
+            foo, bar = cache.get_many("foo", "bar")
+
+        If a key can't be looked up `None` is returned for that key
+        instead.
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        """
         return map(self.get, keys)
 
     def get_dict(self, *keys):
-        return dict(izip(keys, self.get_many(keys)))
+        """Works like :meth:`get_many` but returns a dict::
+
+            d = cache.get_dict("foo", "bar")
+            foo = d["foo"]
+            bar = d["bar"]
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        """
+        return dict(izip(keys, self.get_many(*keys)))
 
     def set(self, key, value, timeout=None):
+        """Adds or overrides a key in the cache.
+
+        :param key: the key to set
+        :param value: the value for the key
+        :param timeout: the cache timeout for the key or the default
+                        timeout if not specified.
+        """
         pass
-    add = set
+
+    def add(self, key, value, timeout=None):
+        """Works like :meth:`set` but does not override already existing
+        values.
+
+        :param key: the key to set
+        :param value: the value for the key
+        :param timeout: the cache timeout for the key or the default
+                        timeout if not specified.
+        """
+        pass
 
     def set_many(self, mapping, timeout=None):
+        """Sets multiple keys and values from a dict.
+
+        :param mapping: a dict with the values to set.
+        :param timeout: the cache timeout for the key or the default
+                        timeout if not specified.
+        """
         for key, value in mapping.iteritems():
             self.set(key, value, timeout)
 
     def delete_many(self, *keys):
+        """Deletes multiple keys at once.
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        """
         for key in keys:
             self.delete(key)
 
     def clear(self):
+        """Clears the cache.  Keep in mind that not all caches support
+        clearning of the full cache.
+        """
         pass
 
     def inc(self, key, delta=1):
+        """Increments the value of a key by `delta`.  If the key does
+        not yet exist it is initialized with `delta`.
+
+        For supporting caches this is an atomic operation.
+
+        :param key: the key to increment.
+        :param delta: the delta to add.
+        """
         self.set(key, (self.get(key) or 0) + delta)
 
     def dec(self, key, delta=1):
+        """Decrements the value of a key by `delta`.  If the key does
+        not yet exist it is initialized with `-delta`.
+
+        For supporting caches this is an atomic operation.
+
+        :param key: the key to increment.
+        :param delta: the delta to subtract.
+        """
         self.set(key, (self.get(key) or 0) - delta)
 
 
 class NullCache(BaseCache):
-    """A cache that doesn't cache."""
+    """A cache that doesn't cache.  This can be useful for unit testing.
+
+    :param default_timeout: a dummy parameter that is ignored but exists
+                            for API compatibility with other caches.
+    """
 
 
 class SimpleCache(BaseCache):
@@ -82,6 +202,11 @@
     mainly for the development server and is not 100% thread safe.  It tries
     to use as many atomic operations as possible and no locks for simplicity
     but it could happen under heavy load that keys are added multiple times.
+
+    :param threshold: the maximum number of items the cache stores before
+                      it starts deleting some.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`.
     """
 
     def __init__(self, threshold=500, default_timeout=300):
@@ -126,33 +251,62 @@
 class MemcachedCache(BaseCache):
     """A cache that uses memcached as backend.
 
+    The first argument can either be a list or tuple of server addresses
+    in which case Werkzeug tries to import the memcache module and connect
+    to it, or an object that resembles the API of a :class:`memcache.Client`.
+
     Implementation notes:  This cache backend works around some limitations in
     memcached to simplify the interface.  For example unicode keys are encoded
-    to utf-8 on the fly.  Methods such as `get_dict` return the keys in the
-    same format as passed.  Furthermore all get methods silently ignore key
-    errors to not cause problems when untrusted user data is passed to the get
-    methods which is often the case in web applications.
+    to utf-8 on the fly.  Methods such as :meth:`~BaseCache.get_dict` return
+    the keys in the same format as passed.  Furthermore all get methods
+    silently ignore key errors to not cause problems when untrusted user data
+    is passed to the get methods which is often the case in web applications.
+
+    :param servers: a list or tuple of server addresses or alternatively
+                    a :class:`memcache.Client` or a compatible client.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`.
+    :param key_prefix: a prefix that is added before all keys.  This makes it
+                       possible to use the same memcached server for different
+                       applications.  Keep in mind that
+                       :meth:`~BaseCache.clear` will also clear keys with a
+                       different prefix.
     """
 
-    def __init__(self, servers, default_timeout=300):
+    def __init__(self, servers, default_timeout=300, key_prefix=None):
         BaseCache.__init__(self, default_timeout)
-        if not have_memcache:
-            raise RuntimeError('no memcache module found')
+        if isinstance(servers, (list, tuple)):
+            try:
+                import cmemcache as memcache
+                is_cmemcache = True
+            except ImportError:
+                try:
+                    import memcache
+                    is_cmemcache = False
+                except ImportError:
+                    raise RuntimeError('no memcache module found')
 
-        # cmemcache has a bug that debuglog is not defined for the
-        # client.  Whenever pickle fails you get a weird AttributError.
-        if is_cmemcache:
-            self._client = memcache.Client(map(str, servers))
-            try:
-                self._client.debuglog = lambda *a: None
-            except:
-                pass
+            # cmemcache has a bug that debuglog is not defined for the
+            # client.  Whenever pickle fails you get a weird AttributError.
+            if is_cmemcache:
+                client = memcache.Client(map(str, servers))
+                try:
+                    client.debuglog = lambda *a: None
+                except:
+                    pass
+            else:
+                client = memcache.Client(servers, False, HIGHEST_PROTOCOL)
         else:
-            self._client = memcache.Client(servers, False, HIGHEST_PROTOCOL)
+            client = servers
+
+        self._client = client
+        self.key_prefix = key_prefix
 
     def get(self, key):
         if isinstance(key, unicode):
             key = key.encode('utf-8')
+        if self.key_prefix:
+            key = self.key_prefix + key
         # memcached doesn't support keys longer than that.  Because often
         # checks for so long keys can occour because it's tested from user
         # submitted data etc we fail silently for getting.
@@ -168,13 +322,15 @@
                 have_encoded_keys = True
             else:
                 encoded_key = key
+            if self.key_prefix:
+                encoded_key = self.key_prefix + encoded_key
             if _test_memcached_key(key):
                 key_mapping[encoded_key] = key
         # the keys call here is important because otherwise cmemcache
         # does ugly things.  What exaclty I don't know, i think it does
         # Py_DECREF but quite frankly i don't care.
         d = rv = self._client.get_multi(key_mapping.keys())
-        if have_encoded_keys:
+        if have_encoded_keys or self.key_prefix:
             rv = {}
             for key, value in d.iteritems():
                 rv[key_mapping[key]] = value
@@ -189,6 +345,8 @@
             timeout = self.default_timeout
         if isinstance(key, unicode):
             key = key.encode('utf-8')
+        if self.key_prefix:
+            key = self.key_prefix + key
         self._client.add(key, value, timeout)
 
     def set(self, key, value, timeout=None):
@@ -196,6 +354,8 @@
             timeout = self.default_timeout
         if isinstance(key, unicode):
             key = key.encode('utf-8')
+        if self.key_prefix:
+            key = self.key_prefix + key
         self._client.set(key, value, timeout)
 
     def get_many(self, *keys):
@@ -209,20 +369,29 @@
         for key, value in mapping.iteritems():
             if isinstance(key, unicode):
                 key = key.encode('utf-8')
+            if self.key_prefix:
+                key = self.key_prefix + key
             new_mapping[key] = value
         self._client.set_multi(new_mapping, timeout)
 
     def delete(self, key):
         if isinstance(key, unicode):
             key = key.encode('utf-8')
-        self._client.delete(key)
+        if self.key_prefix:
+            key = self.key_prefix + key
+        if _test_memcached_key(key):
+            self._client.delete(key)
 
     def delete_many(self, *keys):
-        keys = list(keys)
-        for idx, key in enumerate(keys):
+        new_keys = []
+        for key in keys:
             if isinstance(key, unicode):
-                keys[idx] = key.encode('utf-8')
-        self._client.delete_multi(keys)
+                key = key.encode('utf-8')
+            if self.key_prefix:
+                key = self.key_prefix + key
+            if _test_memcached_key(key):
+                new_keys.append(key)
+        self._client.delete_multi(new_keys)
 
     def clear(self):
         self._client.flush_all()
@@ -230,16 +399,48 @@
     def inc(self, key, delta=1):
         if isinstance(key, unicode):
             key = key.encode('utf-8')
-        self._client.incr(key, key, delta)
+        if self.key_prefix:
+            key = self.key_prefix + key
+        self._client.incr(key, delta)
 
     def dec(self, key, delta=1):
         if isinstance(key, unicode):
             key = key.encode('utf-8')
-        self._client.decr(key, key, delta)
+        if self.key_prefix:
+            key = self.key_prefix + key
+        self._client.decr(key, delta)
+
+
+class GAEMemcachedCache(MemcachedCache):
+    """Connects to the Google appengine memcached Cache.
+
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`.
+    :param key_prefix: a prefix that is added before all keys.  This makes it
+                       possible to use the same memcached server for different
+                       applications.  Keep in mind that
+                       :meth:`~BaseCache.clear` will also clear keys with a
+                       different prefix.
+    """
+
+    def __init__(self, default_timeout=300, key_prefix=None):
+        from google.appengine.api import memcache
+        MemcachedCache.__init__(self, memcache.Client(),
+                                default_timeout, key_prefix)
 
 
 class FileSystemCache(BaseCache):
-    """A cache that stores the items on the file system."""
+    """A cache that stores the items on the file system.  This cache depends
+    on being the only user of the `cache_dir`.  Make absolutely sure that
+    nobody but this cache stores files there or otherwise the chace will
+    randomely delete files therein.
+
+    :param cache_dir: the directory where cached files are stored.
+    :param threshold: the maximum number of items the cache stores before
+                      it starts deleting some.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`.
+    """
 
     def __init__(self, cache_dir, threshold=500, default_timeout=300):
         BaseCache.__init__(self, default_timeout)
@@ -255,7 +456,7 @@
             for idx, key in enumerate(entries):
                 try:
                     f = file(self._get_filename(key))
-                    if pickle.load(f) > now and idx % 3 != 0:
+                    if load(f) > now and idx % 3 != 0:
                         f.close()
                         continue
                 except: