changeset 1637:47a173ae5401

caching module: use_pickle argument for .content() and .update()
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sun, 22 Oct 2006 20:31:37 +0200
parents 985659f4ed55
children 5070d0358544
files MoinMoin/caching.py MoinMoin/i18n/__init__.py MoinMoin/macro/PageHits.py MoinMoin/user.py MoinMoin/wikidicts.py
diffstat 5 files changed, 89 insertions(+), 83 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/caching.py	Sun Oct 22 19:06:15 2006 +0200
+++ b/MoinMoin/caching.py	Sun Oct 22 20:31:37 2006 +0200
@@ -9,6 +9,16 @@
 import os
 import warnings
 
+# cPickle can encode normal and Unicode strings
+# see http://docs.python.org/lib/node66.html
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+# Set pickle protocol, see http://docs.python.org/lib/node64.html
+PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
+
 from MoinMoin import config
 from MoinMoin.util import filesys, lock
 
@@ -16,6 +26,10 @@
 # where only we should have write access initially
 warnings.filterwarnings("ignore", "tempnam.*security", RuntimeWarning, "MoinMoin.caching")
 
+class CacheError(Exception):
+    """ raised if we have trouble reading or writing to the cache """
+    pass
+
 class CacheEntry:
     def __init__(self, request, arena, key, scope='page_or_wiki', do_locking=True):
         """ init a cache entry
@@ -104,26 +118,31 @@
         else:
             self.request.log("Can't acquire write lock in %s" % self.lock_dir)
 
-    def update(self, content, encode=False):
-        tmpfname = self._tmpfilename()
-        fname = self._filename()
-        if encode:
-            content = content.encode(config.charset)
-        if not self.locking or self.locking and self.wlock.acquire(1.0):
-            try:
-                # we do not write content to old inode, but to a new file
-                # se we don't need to lock when we just want to read the file
-                # (at least on POSIX, this works)
-                f = open(tmpfname, 'wb')
-                f.write(content)
-                f.close()
-                # this is either atomic or happening with real locks set:
-                filesys.rename(tmpfname, fname)
-            finally:
-                if self.locking:
-                    self.wlock.release()
-        else:
-            self.request.log("Can't acquire write lock in %s" % self.lock_dir)
+    def update(self, content, encode=False, use_pickle=False):
+        try:
+            tmpfname = self._tmpfilename()
+            fname = self._filename()
+            if encode:
+                content = content.encode(config.charset)
+            elif use_pickle:
+                content = pickle.dumps(content, PICKLE_PROTOCOL)
+            if not self.locking or self.locking and self.wlock.acquire(1.0):
+                try:
+                    # we do not write content to old inode, but to a new file
+                    # se we don't need to lock when we just want to read the file
+                    # (at least on POSIX, this works)
+                    f = open(tmpfname, 'wb')
+                    f.write(content)
+                    f.close()
+                    # this is either atomic or happening with real locks set:
+                    filesys.rename(tmpfname, fname)
+                finally:
+                    if self.locking:
+                        self.wlock.release()
+            else:
+                self.request.log("Can't acquire write lock in %s" % self.lock_dir)
+        except (pickle.PicklingError, IOError, ValueError), err:
+            raise CacheError(str(err))
 
     def remove(self):
         if not self.locking or self.locking and self.wlock.acquire(1.0):
@@ -138,18 +157,23 @@
         else:
             self.request.log("Can't acquire write lock in %s" % self.lock_dir)
 
-    def content(self, decode=False):
-        if not self.locking or self.locking and self.rlock.acquire(1.0):
-            try:
-                f = open(self._filename(), 'rb')
-                data = f.read()
-                f.close()
-            finally:
-                if self.locking:
-                    self.rlock.release()
-        else:
-            self.request.log("Can't acquire read lock in %s" % self.lock_dir)
-        if decode:
-            data = data.decode(config.charset)
-        return data
+    def content(self, decode=False, use_pickle=False):
+        try:
+            if not self.locking or self.locking and self.rlock.acquire(1.0):
+                try:
+                    f = open(self._filename(), 'rb')
+                    data = f.read()
+                    f.close()
+                finally:
+                    if self.locking:
+                        self.rlock.release()
+            else:
+                self.request.log("Can't acquire read lock in %s" % self.lock_dir)
+            if decode:
+                data = data.decode(config.charset)
+            elif use_pickle:
+                data = pickle.loads(data)
+            return data
+        except (pickle.UnpicklingError, IOError, EOFError, ValueError), err:
+            raise CacheError(str(err))
 
--- a/MoinMoin/i18n/__init__.py	Sun Oct 22 19:06:15 2006 +0200
+++ b/MoinMoin/i18n/__init__.py	Sun Oct 22 20:31:37 2006 +0200
@@ -28,13 +28,6 @@
 
 import os, gettext, glob
 
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-# Set pickle protocol, see http://docs.python.org/lib/node64.html
-PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
-
 from MoinMoin import caching
 
 # This is a global for a reason: in persistent environments all languages in
@@ -80,12 +73,18 @@
                 for key, value in t.info.items():
                     #request.log("meta key %s value %r" % (key, value))
                     _languages[language][key] = value.decode(encoding)
-            meta_cache.update(pickle.dumps(_languages))
+            try:
+                meta_cache.update(_languages, use_pickle=True)
+            except caching.CacheError:
+                pass
 
-    if languages is None: # another tread maybe has done it before us
-        _languages = pickle.loads(meta_cache.content())
-        if languages is None:
-            languages = _languages
+    if languages is None: # another thread maybe has done it before us
+        try:
+            _languages = meta_cache.content(use_pickle=True)
+            if languages is None:
+                languages = _languages
+        except caching.CacheError:
+            pass
     request.clock.stop('i18n_init')
 
 
@@ -172,8 +171,8 @@
             request.log("i18n: langfilename %s needsupdate %d" % (langfilename, needsupdate))
         if not needsupdate:
             try:
-                uc_texts, uc_unformatted = pickle.loads(cache.content())
-            except (IOError, ValueError, pickle.UnpicklingError): # bad pickle data, no pickle
+                uc_texts, uc_unformatted = cache.content(use_pickle=True)
+            except caching.CacheError:
                 if debug:
                     request.log("i18n: pickle %s load failed" % lang)
                 needsupdate = 1
@@ -202,7 +201,10 @@
                         uc_texts[ukey] = u"%s*" % utext
             if debug:
                 request.log("i18n: dumping lang %s" % lang)
-            cache.update(pickle.dumps((uc_texts, uc_unformatted), PICKLE_PROTOCOL))
+            try:
+                cache.update((uc_texts, uc_unformatted), use_pickle=True)
+            except caching.CacheError:
+                pass
 
         self.formatted = uc_texts
         self.raw = uc_unformatted
--- a/MoinMoin/macro/PageHits.py	Sun Oct 22 19:06:15 2006 +0200
+++ b/MoinMoin/macro/PageHits.py	Sun Oct 22 20:31:37 2006 +0200
@@ -7,14 +7,6 @@
 
 """
 
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-# Set pickle protocol, see http://docs.python.org/lib/node64.html
-PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
-
 from MoinMoin import caching, config, logfile
 from MoinMoin.Page import Page
 from MoinMoin.logfile import eventlog
@@ -42,8 +34,8 @@
         date, hits = 0, {}
         if self.cache.exists():
             try:
-                date, hits = pickle.loads(self.cache.content())
-            except (pickle.UnpicklingError, IOError, EOFError, ValueError):
+                date, hits = self.cache.content(use_pickle=True)
+            except caching.CacheError:
                 self.cache.remove()
         return date, hits
 
@@ -69,7 +61,10 @@
             self.updateCache(logDate, hits)
 
     def updateCache(self, date, hits):
-        self.cache.update(pickle.dumps((date, hits), PICKLE_PROTOCOL))
+        try:
+            self.cache.update((date, hits), use_pickle=True)
+        except caching.CacheError:
+            pass
 
     def filterReadableHits(self, hits):
         """ Filter out hits the user many not see """
--- a/MoinMoin/user.py	Sun Oct 22 19:06:15 2006 +0200
+++ b/MoinMoin/user.py	Sun Oct 22 20:31:37 2006 +0200
@@ -11,14 +11,6 @@
 
 import os, time, sha, codecs
 
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-# Set pickle protocol, see http://docs.python.org/lib/node64.html
-PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
-
 from MoinMoin import config, caching, wikiutil, i18n
 from MoinMoin.util import filesys, timefuncs
 
@@ -62,8 +54,8 @@
         key = 'name2id'
         cache = caching.CacheEntry(request, arena, key, scope='wiki')
         try:
-            _name2id = pickle.loads(cache.content())
-        except (pickle.UnpicklingError, IOError, EOFError, ValueError):
+            _name2id = cache.content(use_pickle=True)
+        except caching.CacheError:
             _name2id = {}
         cfg.cache.name2id = _name2id
     id = _name2id.get(searchName, None)
@@ -75,7 +67,10 @@
         arena = 'user'
         key = 'name2id'
         cache = caching.CacheEntry(request, arena, key, scope='wiki')
-        cache.update(pickle.dumps(_name2id, PICKLE_PROTOCOL))
+        try:
+            cache.update(_name2id, use_pickle=True)
+        except caching.CacheError:
+            pass
         id = _name2id.get(searchName, None)
     return id
 
--- a/MoinMoin/wikidicts.py	Sun Oct 22 19:06:15 2006 +0200
+++ b/MoinMoin/wikidicts.py	Sun Oct 22 20:31:37 2006 +0200
@@ -11,16 +11,6 @@
 #import copy #broken, see comments at top of this file:
 from MoinMoin.support import copy
 
-# cPickle can encode normal and Unicode strings
-# see http://docs.python.org/lib/node66.html
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-# Set pickle protocol, see http://docs.python.org/lib/node64.html
-PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
-
 from MoinMoin import config, caching, wikiutil, Page, logfile
 from MoinMoin.logfile.editlog import EditLog
 
@@ -326,7 +316,7 @@
         except AttributeError:
             try:
                 cache = caching.CacheEntry(request, arena, key, scope='wiki')
-                data = pickle.loads(cache.content())
+                data = cache.content(use_pickle=True)
                 self.__dict__.update(data)
 
                 # invalidate the cache if the pickle version changed
@@ -412,7 +402,7 @@
                 self.dictdict[name].expandgroups(self)
 
             cache = caching.CacheEntry(request, arena, key, scope='wiki')
-            cache.update(pickle.dumps(data, PICKLE_PROTOCOL))
+            cache.update(data, use_pickle=True)
 
         # remember it (persistent environments)
         self.cfg.cache.DICTS_DATA = data