changeset 3878:b13a58a18dac

caching module backported from 1.8 (file-like new api, old api still as is)
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sat, 19 Jul 2008 14:36:38 +0200
parents c4cf4327c96e
children f042906b346e 85cd05b8af42
files MoinMoin/_tests/test_caching.py MoinMoin/caching.py
diffstat 2 files changed, 161 insertions(+), 59 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/_tests/test_caching.py	Fri Jul 18 19:50:15 2008 +0200
+++ b/MoinMoin/_tests/test_caching.py	Sat Jul 19 14:36:38 2008 +0200
@@ -76,6 +76,24 @@
         page._write_file(test_data2)
         assert cache.needsUpdate(page._text_filename())
 
+    def test_filelike_readwrite(self):
+        request = self.request
+        key = 'nooneknowsit'
+        arena = 'somethingfunny'
+        data = "dontcare"
+        cacheentry = caching.CacheEntry(request, arena, key, scope='wiki', do_locking=True,
+                 use_pickle=False, use_encode=True)
+        cacheentry.open(mode='w')
+        cacheentry.write(data)
+        cacheentry.close()
+
+        assert cacheentry.exists()
+
+        cacheentry.open(mode='r')
+        rdata = cacheentry.read()
+        cacheentry.close()
+
+        assert data == rdata
 
 coverage_modules = ['MoinMoin.caching']
 
--- a/MoinMoin/caching.py	Fri Jul 18 19:50:15 2008 +0200
+++ b/MoinMoin/caching.py	Sat Jul 19 14:36:38 2008 +0200
@@ -3,11 +3,13 @@
     MoinMoin caching module
 
     @copyright: 2001-2004 by Juergen Hermann <jh@web.de>,
-                2006-2008 MoinMoin:ThomasWaldmann
+                2006-2008 MoinMoin:ThomasWaldmann,
+                2008 MoinMoin:ThomasPfaff
     @license: GNU GPL, see COPYING for details.
 """
 
 import os
+import shutil
 import tempfile
 
 from MoinMoin import log
@@ -21,6 +23,7 @@
     """ raised if we have trouble reading or writing to the cache """
     pass
 
+
 def get_arena_dir(request, arena, scope):
     if scope == 'page_or_wiki': # XXX DEPRECATED, remove later
         if isinstance(arena, str):
@@ -69,22 +72,38 @@
         self.arena_dir = get_arena_dir(request, arena, scope)
         if not os.path.exists(self.arena_dir):
             os.makedirs(self.arena_dir)
+        self._fname = os.path.join(self.arena_dir, key)
+
         if self.locking:
             self.lock_dir = os.path.join(self.arena_dir, '__lock__')
             self.rlock = lock.LazyReadLock(self.lock_dir, 60.0)
             self.wlock = lock.LazyWriteLock(self.lock_dir, 60.0)
 
+        # used by file-like api:
+        self._lock = None  # either self.rlock or self.wlock
+        self._fileobj = None  # open cache file object
+        self._tmp_fname = None  # name of temporary file (used for write)
+        self._mode = None  # mode of open file object
+
+
     def _filename(self):
-        return os.path.join(self.arena_dir, self.key)
+        # DEPRECATED - please use file-like api
+        return self._fname
 
     def exists(self):
-        return os.path.exists(self._filename())
+        return os.path.exists(self._fname)
 
     def mtime(self):
         # DEPRECATED for checking a changed on-disk cache, please use
         # self.uid() for this, see below
         try:
-            return os.path.getmtime(self._filename())
+            return os.path.getmtime(self._fname)
+        except (IOError, OSError):
+            return 0
+
+    def size(self):
+        try:
+            return os.path.getsize(self._fname)
         except (IOError, OSError):
             return 0
 
@@ -93,7 +112,7 @@
 
             See docstring of MoinMoin.util.filesys.fuid for details.
         """
-        return filesys.fuid(self._filename())
+        return filesys.fuid(self._fname)
 
     def needsUpdate(self, filename, attachdir=None):
         # following code is not necessary. will trigger exception and give same result
@@ -101,7 +120,7 @@
         #    return 1
 
         try:
-            ctime = os.path.getmtime(self._filename())
+            ctime = os.path.getmtime(self._fname)
             ftime = os.path.getmtime(filename)
         except os.error:
             return 1
@@ -118,53 +137,137 @@
 
         return needsupdate
 
-#    def copyto(self, filename):
-#        # currently unused function
-#        import shutil
-#        tmpfname = self._tmpfilename()
-#        fname = self._filename()
-#        if not self.locking or self.locking and self.wlock.acquire(1.0):
-#            try:
-#                shutil.copyfile(filename, tmpfname)
-#                # this is either atomic or happening with real locks set:
-#                filesys.rename(tmpfname, fname)
-#            finally:
-#                if self.locking:
-#                    self.wlock.release()
-#        else:
-#            logging.error("Can't acquire write lock in %s" % self.lock_dir)
+    def _determine_locktype(self, mode):
+        """ return the correct lock object for a specific file access mode """
+        if self.locking:
+            if 'r' in mode:
+                lock = self.rlock
+            if 'w' in mode or 'a' in mode:
+                lock = self.wlock
+        else:
+            lock = None
+        return lock
+
+    # file-like interface ----------------------------------------------------
+
+    def open(self, filename=None, mode='r', bufsize=-1):
+        """ open the cache for reading/writing
+
+        @param filename: must be None (default - automatically determine filename)
+        @param mode: 'r' (read, default), 'w' (write)
+                     Note: if mode does not include 'b' (binary), it will be
+                           automatically changed to include 'b'.
+        @param bufsize: size of read/write buffer (default: -1 meaning automatic)
+        @return: None (the opened file object is kept in self._fileobj and used
+                 implicitely by read/write/close functions of CacheEntry object.
+        """
+        assert self._fileobj is None, 'caching: trying to open an already opened cache'
+        assert filename is None, 'caching: giving a filename is not supported (yet?)'
+
+        self._lock = self._determine_locktype(mode)
+
+        if 'b' not in mode:
+            mode += 'b'  # we want to use binary mode, ever!
+        self._mode = mode  # for self.close()
+
+        if not self.locking or self.locking and self._lock.acquire(1.0):
+            try:
+                if 'r' in mode:
+                    self._fileobj = open(self._fname, mode, bufsize)
+                elif 'w' in mode:
+                    # we do not write content to old inode, but to a new file
+                    # so we don't need to lock when we just want to read the file
+                    # (at least on POSIX, this works)
+                    fd, self._tmp_fname = tempfile.mkstemp('.tmp', self.key, self.arena_dir)
+                    self._fileobj = os.fdopen(fd, mode, bufsize)
+                else:
+                    raise ValueError("caching: mode does not contain 'r' or 'w'")
+            finally:
+                if self.locking:
+                    self._lock.release()
+                    self._lock = None
+        else:
+            logging.error("Can't acquire read/write lock in %s" % self.lock_dir)
+
+
+    def read(self, size=-1):
+        """ read data from cache file
+
+        @param size: how many bytes to read (default: -1 == everything)
+        @return: read data (str)
+        """
+        return self._fileobj.read(size)
+
+    def write(self, data):
+        """ write data to cache file
+
+        @param data: write data (str)
+        """
+        self._fileobj.write(data)
+
+    def close(self):
+        """ close cache file (and release lock, if any) """
+        if self._fileobj:
+            self._fileobj.close()
+            self._fileobj = None
+            if 'w' in self._mode:
+                filesys.chmod(self._tmp_fname, 0666 & config.umask) # fix mode that mkstemp chose
+                # this is either atomic or happening with real locks set:
+                filesys.rename(self._tmp_fname, self._fname)
+
+        if self._lock:
+            if self.locking:
+                self._lock.release()
+            self._lock = None
+
+    # ------------------------------------------------------------------------
 
     def update(self, content):
         try:
-            fname = self._filename()
-            if self.use_pickle:
-                content = pickle.dumps(content, PICKLE_PROTOCOL)
-            elif self.use_encode:
-                content = content.encode(config.charset)
-            if not self.locking or self.locking and self.wlock.acquire(1.0):
+            if hasattr(content, 'read'):
+                # content is file-like
+                assert not (self.use_pickle or self.use_encode), 'caching: use_pickle and use_encode not supported with file-like api'
                 try:
-                    # we do not write content to old inode, but to a new file
-                    # so we don't need to lock when we just want to read the file
-                    # (at least on POSIX, this works)
-                    tmp_handle, tmp_fname = tempfile.mkstemp('.tmp', self.key, self.arena_dir)
-                    os.write(tmp_handle, content)
-                    os.close(tmp_handle)
-                    # this is either atomic or happening with real locks set:
-                    filesys.rename(tmp_fname, fname)
-                    filesys.chmod(fname, 0666 & config.umask) # fix mode that mkstemp chose
+                    self.open(mode='w')
+                    shutil.copyfileobj(content, self)
                 finally:
-                    if self.locking:
-                        self.wlock.release()
+                    self.close()
             else:
-                logging.error("Can't acquire write lock in %s" % self.lock_dir)
+                # content is a string
+                if self.use_pickle:
+                    content = pickle.dumps(content, PICKLE_PROTOCOL)
+                elif self.use_encode:
+                    content = content.encode(config.charset)
+
+                try:
+                    self.open(mode='w')
+                    self.write(content)
+                finally:
+                    self.close()
         except (pickle.PicklingError, OSError, IOError, ValueError), err:
             raise CacheError(str(err))
 
+    def content(self):
+        # no file-like api yet, we implement it when we need it
+        try:
+            try:
+                self.open(mode='r')
+                data = self.read()
+            finally:
+                self.close()
+            if self.use_pickle:
+                data = pickle.loads(data)
+            elif self.use_encode:
+                data = data.decode(config.charset)
+            return data
+        except (pickle.UnpicklingError, IOError, EOFError, ValueError), err:
+            raise CacheError(str(err))
+
     def remove(self):
         if not self.locking or self.locking and self.wlock.acquire(1.0):
             try:
                 try:
-                    os.remove(self._filename())
+                    os.remove(self._fname)
                 except OSError:
                     pass
             finally:
@@ -173,23 +276,4 @@
         else:
             logging.error("Can't acquire write lock in %s" % self.lock_dir)
 
-    def content(self):
-        try:
-            if not self.locking or self.locking and self.rlock.acquire(1.0):
-                try:
-                    f = open(self._filename(), 'rb')
-                    data = f.read()
-                    f.close()
-                finally:
-                    if self.locking:
-                        self.rlock.release()
-            else:
-                logging.error("Can't acquire read lock in %s" % self.lock_dir)
-            if self.use_pickle:
-                data = pickle.loads(data)
-            elif self.use_encode:
-                data = data.decode(config.charset)
-            return data
-        except (pickle.UnpicklingError, IOError, EOFError, ValueError), err:
-            raise CacheError(str(err))