view MoinMoin/ @ 3860:c18d557b5aec

caching: remember lock in self._lock, handle it in a similar way as self._fileobj
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Wed, 16 Jul 2008 17:30:43 +0200
parents 8cb2f4ebd45f
children be61d38685f5
line wrap: on
line source
# -*- coding: iso-8859-1 -*-
    MoinMoin caching module

    @copyright: 2001-2004 by Juergen Hermann <>,
                2006-2008 MoinMoin:ThomasWaldmann
    @license: GNU GPL, see COPYING for details.

import os
import tempfile

from MoinMoin import log
logging = log.getLogger(__name__)

from MoinMoin import config
from MoinMoin.util import filesys, lock, pickle, PICKLE_PROTOCOL

class CacheError(Exception):
    """ raised if we have trouble reading or writing to the cache """

def get_arena_dir(request, arena, scope):
    if scope == 'page_or_wiki': # XXX DEPRECATED, remove later
        if isinstance(arena, str):
            return os.path.join(request.cfg.cache_dir, request.cfg.siteid, arena)
        else: # arena is in fact a page object
            return arena.getPagePath('cache', check_create=1)
    elif scope == 'item': # arena is a Page instance
        # we could move cache out of the page directory and store it to cache_dir
        return arena.getPagePath('cache', check_create=1)
    elif scope == 'wiki':
        return os.path.join(request.cfg.cache_dir, request.cfg.siteid, arena)
    elif scope == 'farm':
        return os.path.join(request.cfg.cache_dir, '__common__', arena)
    return None

def get_cache_list(request, arena, scope):
    arena_dir = get_arena_dir(request, arena, scope)
        return filesys.dclistdir(arena_dir)
    except OSError:
        return []

class CacheEntry:
    def __init__(self, request, arena, key, scope='page_or_wiki', do_locking=True,
                 use_pickle=False, use_encode=False):
        """ init a cache entry
            @param request: the request object
            @param arena: either a string or a page object, when we want to use
                          page local cache area
            @param key: under which key we access the cache content
            @param scope: the scope where we are caching:
                          'item' - an item local cache
                          'wiki' - a wiki local cache
                          'farm' - a cache for the whole farm
            @param do_locking: if there should be a lock, normally True
            @param use_pickle: if data should be pickled/unpickled (nice for arbitrary cache content)
            @param use_encode: if data should be encoded/decoded (nice for readable cache files)
        self.request = request
        self.key = key
        self.locking = do_locking
        self.use_pickle = use_pickle
        self.use_encode = use_encode
        self.arena_dir = get_arena_dir(request, arena, scope)
        if not os.path.exists(self.arena_dir):
        if self.locking:
            self.lock_dir = os.path.join(self.arena_dir, '__lock__')
            self.rlock = lock.LazyReadLock(self.lock_dir, 60.0)
            self.wlock = lock.LazyWriteLock(self.lock_dir, 60.0)
        # used by file-like api:
        self._fileobj = None
        self._lock = None

    def _filename(self):
        return os.path.join(self.arena_dir, self.key)

    def exists(self):
        return os.path.exists(self._filename())

    def mtime(self):
        # DEPRECATED for checking a changed on-disk cache, please use
        # self.uid() for this, see below
            return os.path.getmtime(self._filename())
        except (IOError, OSError):
            return 0

    def uid(self):
        """ Return a value that likely changes when the on-disk cache was updated.

            See docstring of MoinMoin.util.filesys.fuid for details.
        return filesys.fuid(self._filename())

    def needsUpdate(self, filename, attachdir=None):
        # following code is not necessary. will trigger exception and give same result
        #if not self.exists():
        #    return 1

            ctime = os.path.getmtime(self._filename())
            ftime = os.path.getmtime(filename)
        except os.error:
            return 1

        needsupdate = ftime > ctime

        # if a page depends on the attachment dir, we check this, too:
        if not needsupdate and attachdir:
                ftime2 = os.path.getmtime(attachdir)
            except os.error:
                ftime2 = 0
            needsupdate = ftime2 > ctime

        return needsupdate

#    def copyto(self, filename):
#        # currently unused function
#        import shutil
#        tmpfname = self._tmpfilename()
#        fname = self._filename()
#        if not self.locking or self.locking and self.wlock.acquire(1.0):
#            try:
#                shutil.copyfile(filename, tmpfname)
#                # this is either atomic or happening with real locks set:
#                filesys.rename(tmpfname, fname)
#            finally:
#                if self.locking:
#                    self.wlock.release()
#        else:
#            logging.error("Can't acquire write lock in %s" % self.lock_dir)

    def _determine_locktype(self, mode):
        #returns the correct locker object for a specific file access mode
        if 'r' in mode:
            lock = self.rlock
        if 'w' in mode or 'a' in mode:
            lock = self.wlock
        return lock

    # file-like interface ----------------------------------------------------

    def open(self, filename=None, mode='r', bufsize=-1):
        if self._fileobj:
            # bug-possibility: this doesn't check, if there is any lock on the file
            # we assume this, as the first call to open should
            # acquire the lock.
        if filename is None:
            filename = self._filename()
            raise Exception('caching: giving a filename is not supported (yet?)')

        #acquire the correct lock for the desired mode
        self._lock = self._determine_locktype(mode)

        if not self.locking or self.locking and self._lock.acquire(1.0):
                self._fileobj = open(filename, mode, bufsize)
            except IOError:
                if self.locking:
                logging.error("Can't open cache file %s" % filename)
            logging.error("Can't acquire read/write lock in %s" % self.lock_dir)

    def read(self, size=-1):

    def write(self, data):

    def close(self):
        if self._fileobj:
            self._fileobj = None

        if self._lock:
            if self.locking:
            self._lock = None

    # ------------------------------------------------------------------------

    def update(self, content):
            fname = self._filename()
            if self.use_pickle:
                content = pickle.dumps(content, PICKLE_PROTOCOL)
            elif self.use_encode:
                content = content.encode(config.charset)
            if not self.locking or self.locking and self.wlock.acquire(1.0):
                    # we do not write content to old inode, but to a new file
                    # so we don't need to lock when we just want to read the file
                    # (at least on POSIX, this works)
                    tmp_handle, tmp_fname = tempfile.mkstemp('.tmp', self.key, self.arena_dir)
                    os.write(tmp_handle, content)
                    # this is either atomic or happening with real locks set:
                    filesys.rename(tmp_fname, fname)
                    filesys.chmod(fname, 0666 & config.umask) # fix mode that mkstemp chose
                    if self.locking:
                logging.error("Can't acquire write lock in %s" % self.lock_dir)
        except (pickle.PicklingError, OSError, IOError, ValueError), err:
            raise CacheError(str(err))

    def remove(self):
        if not self.locking or self.locking and self.wlock.acquire(1.0):
                except OSError:
                if self.locking:
            logging.error("Can't acquire write lock in %s" % self.lock_dir)

    def content(self):
            if not self.locking or self.locking and self.rlock.acquire(1.0):
                    f = open(self._filename(), 'rb')
                    data =
                    if self.locking:
                logging.error("Can't acquire read lock in %s" % self.lock_dir)
            if self.use_pickle:
                data = pickle.loads(data)
            elif self.use_encode:
                data = data.decode(config.charset)
            return data
        except (pickle.UnpicklingError, IOError, EOFError, ValueError), err:
            raise CacheError(str(err))