view MoinMoin/ @ 6087:538b7d0119cd

tagged release 1.9.8
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Fri, 17 Oct 2014 21:53:46 +0200
parents 7e3e406f21d9
line wrap: on
line source

# -*- coding: iso-8859-1 -*-
    MoinMoin caching module

    @copyright: 2001-2004 by Juergen Hermann <>,
                2006-2009 MoinMoin:ThomasWaldmann,
                2008 MoinMoin:ThomasPfaff
    @license: GNU GPL, see COPYING for details.

import os
import shutil
import tempfile

from MoinMoin import log
logging = log.getLogger(__name__)

from MoinMoin import config
from MoinMoin.util import filesys, lock, pickle, PICKLE_PROTOCOL

class CacheError(Exception):
    """ raised if we have trouble locking, reading or writing """

def get_arena_dir(request, arena, scope):
    if scope == 'item': # arena is a Page instance
        # we could move cache out of the page directory and store it to cache_dir
        return arena.getPagePath('cache', check_create=1)
    elif scope == 'wiki':
        return os.path.join(request.cfg.cache_dir, request.cfg.siteid, arena)
    elif scope == 'farm':
        return os.path.join(request.cfg.cache_dir, '__common__', arena)
    elif scope == 'userdir':
        return os.path.join(request.cfg.user_dir, 'cache', arena)
    elif scope == 'dir':
        # arena is a specific directory, just use it
        return arena
    return None

def get_cache_list(request, arena, scope):
    arena_dir = get_arena_dir(request, arena, scope)
        return os.listdir(arena_dir)
    except OSError:
        return []

class CacheEntry:
    def __init__(self, request, arena, key, scope='wiki', do_locking=True,
                 use_pickle=False, use_encode=False):
        """ init a cache entry
            @param request: the request object
            @param arena: either a string or a page object, when we want to use
                          page local cache area
            @param key: under which key we access the cache content
            @param scope: the scope where we are caching:
                          'item' - an item local cache
                          'wiki' - a wiki local cache
                          'farm' - a cache for the whole farm
                          'dir' - just use some specific directory
            @param do_locking: if there should be a lock, normally True
            @param use_pickle: if data should be pickled/unpickled (nice for arbitrary cache content)
            @param use_encode: if data should be encoded/decoded (nice for readable cache files)
        self.request = request
        self.key = key
        self.locking = do_locking
        self.use_pickle = use_pickle
        self.use_encode = use_encode
        self.arena_dir = get_arena_dir(request, arena, scope)
        if not os.path.exists(self.arena_dir):
        self._fname = os.path.join(self.arena_dir, key)

        # used by file-like api:
        self._lock = None  # either a read or a write lock
        self._fileobj = None  # open cache file object
        self._tmp_fname = None  # name of temporary file (used for write)
        self._mode = None  # mode of open file object

    def _filename(self):
        # DEPRECATED - please use file-like api
        return self._fname

    def exists(self):
        return os.path.exists(self._fname)

    def mtime(self):
        # DEPRECATED for checking a changed on-disk cache, please use
        # self.uid() for this, see below
            return os.path.getmtime(self._fname)
        except (IOError, OSError):
            return 0

    def size(self):
            return os.path.getsize(self._fname)
        except (IOError, OSError):
            return 0

    def uid(self):
        """ Return a value that likely changes when the on-disk cache was updated.

            See docstring of MoinMoin.util.filesys.fuid for details.
        return filesys.fuid(self._fname)

    def needsUpdate(self, filename, attachdir=None):
        # following code is not necessary. will trigger exception and give same result
        #if not self.exists():
        #    return 1

            ctime = os.path.getmtime(self._fname)
            ftime = os.path.getmtime(filename)
        except os.error:
            return 1

        needsupdate = ftime > ctime

        # if a page depends on the attachment dir, we check this, too:
        if not needsupdate and attachdir:
                ftime2 = os.path.getmtime(attachdir)
            except os.error:
                ftime2 = 0
            needsupdate = ftime2 > ctime

        return needsupdate

    def lock(self, mode, timeout=10.0):
        acquire a lock for <mode> ("r" or "w").
        we just raise a CacheError if this doesn't work.

         * .open() calls .lock(), .close() calls .unlock() if do_locking is True.
         * if you need to do a read-modify-write, you want to use a CacheEntry
           with do_locking=False and manually call .lock('w') and .unlock().
        lock_dir = os.path.join(self.arena_dir, '__lock__')
        if 'r' in mode:
            _lock = lock.LazyReadLock(lock_dir, 60.0)
        elif 'w' in mode:
            _lock = lock.LazyWriteLock(lock_dir, 60.0)
        acquired = _lock.acquire(timeout)
        if acquired:
            self._lock = _lock
            self._lock = None
            err = "Can't acquire %s lock in %s" % (mode, lock_dir)
            raise CacheError(err)

    def unlock(self):
        release the lock.
        if self._lock:
            self._lock = None

    # file-like interface ----------------------------------------------------

    def open(self, filename=None, mode='r', bufsize=-1):
        """ open the cache for reading/writing

        Typical usage:
      'r')  # open file, create locks
                data =
                cache.close()  # important to close file and remove locks

        @param filename: must be None (default - automatically determine filename)
        @param mode: 'r' (read, default), 'w' (write)
                     Note: if mode does not include 'b' (binary), it will be
                           automatically changed to include 'b'.
        @param bufsize: size of read/write buffer (default: -1 meaning automatic)
        @return: None (the opened file object is kept in self._fileobj and used
                 implicitely by read/write/close functions of CacheEntry object.
        assert self._fileobj is None, 'caching: trying to open an already opened cache'
        assert filename is None, 'caching: giving a filename is not supported (yet?)'
        assert 'r' in mode or 'w' in mode, 'caching: mode must contain "r" or "w"'

        if 'b' not in mode:
            mode += 'b'  # we want to use binary mode, ever!
        self._mode = mode  # for self.close()

        if self.locking:
            if 'r' in mode:
                filename = self._fname
                self._fileobj = open(filename, mode, bufsize)
            elif 'w' in mode:
                # we do not write content to old inode, but to a new file
                # so we don't need to lock when we just want to read the file
                # (at least on POSIX, this works)
                filename = None
                fd, filename = tempfile.mkstemp('.tmp', self.key, self.arena_dir)
                self._tmp_fname = filename
                self._fileobj = os.fdopen(fd, mode, bufsize)
        except IOError, err:
            if 'w' in mode:
                # IOerror for 'r' can be just a non-existing file, do not log that,
                # but if open fails for 'w', we likely have some bigger problem:
            raise CacheError(str(err))

    def read(self, size=-1):
        """ read data from cache file

        @param size: how many bytes to read (default: -1 == everything)
        @return: read data (str)

    def write(self, data):
        """ write data to cache file

        @param data: write data (str)

    def close(self):
        """ close cache file (and release lock, if any) """
            if self._fileobj:
                self._fileobj = None
                if 'w' in self._mode:
                    filesys.chmod(self._tmp_fname, 0666 & config.umask) # fix mode that mkstemp chose
                    # this is either atomic or happening with real locks set:
                    filesys.rename(self._tmp_fname, self._fname)
            if self.locking:

    # ------------------------------------------------------------------------

    def update(self, content):
            if hasattr(content, 'read'):
                # content is file-like
                assert not (self.use_pickle or self.use_encode), 'caching: use_pickle and use_encode not supported with file-like api'
                    shutil.copyfileobj(content, self)
                # content is a string
                if self.use_pickle:
                    content = pickle.dumps(content, PICKLE_PROTOCOL)
                elif self.use_encode:
                    content = content.encode(config.charset)

        except (pickle.PicklingError, OSError, IOError, ValueError), err:
            raise CacheError(str(err))

    def content(self):
        # no file-like api yet, we implement it when we need it
                data =
            if self.use_pickle:
                data = pickle.loads(data)
            elif self.use_encode:
                data = data.decode(config.charset)
            return data
        except (pickle.UnpicklingError, IOError, EOFError, ValueError), err:
            raise CacheError(str(err))

    def remove(self):
        if self.locking:
            except OSError:
            if self.locking: