changeset 3609:9653e7a9bcc4

new function filesys.fuid(fname) as slightly better file mtime replacement + tests
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sat, 17 May 2008 19:54:30 +0200
parents 3d5985a65342
children 39456a5dc4e8
files MoinMoin/util/_tests/test_filesys.py MoinMoin/util/filesys.py
diffstat 2 files changed, 139 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_filesys.py	Sat May 17 19:54:30 2008 +0200
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+    MoinMoin - MoinMoin.util.filesys Tests
+
+    @copyright: 2008 MoinMoin:ThomasWaldmann
+    @license: GNU GPL, see COPYING for details.
+"""
+import sys, os, time
+import shutil, tempfile
+
+import py.test
+
+from MoinMoin.util import filesys
+
+class TestFuid:
+    """ test filesys.fuid (a better mtime() alternative for up-to-date checking) """
+
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'fuid_')
+        self.fname = os.path.join(self.test_dir, "fuid-test")
+        self.tmpname = os.path.join(self.test_dir, "fuid-temp")
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+
+    def testNoFile(self):
+        # no file created
+        uid = filesys.fuid(self.fname)
+
+        assert uid is None  # there is no file yet, fuid will fail internally and return None
+
+    def makefile(self, fname, content):
+        f = open(fname, "w")
+        f.write(content)
+        f.close()
+
+    def testNewFile(self):
+        # freshly created file
+        self.makefile(self.fname, "foo")
+        uid1 = filesys.fuid(self.fname)
+
+        assert uid1 is not None  # None would mean some failure in fuid()
+
+    def testUpdateFileInPlace(self):
+        # update file in place, changing size and maybe mtime
+        self.makefile(self.fname, "foo")
+        uid1 = filesys.fuid(self.fname)
+
+        self.makefile(self.fname, "foofoo")
+        uid2 = filesys.fuid(self.fname)
+
+        assert uid2 != uid1 # we changed size and maybe mtime
+
+    def testUpdateFileMovingFromTemp(self):
+        # update file by moving another file over it (see caching.update)
+        # changing inode, maybe mtime, but not size
+        if sys.platform == 'win32':
+            py.test.skip("Inode change detection not supported on win32")
+
+        self.makefile(self.fname, "foo")
+        uid1 = filesys.fuid(self.fname)
+
+        self.makefile(self.tmpname, "bar")
+        os.rename(self.tmpname, self.fname)
+        uid2 = filesys.fuid(self.fname)
+
+        assert uid2 != uid1 # we didn't change size, but inode and maybe mtime
+
+    def testStale(self):
+        # is a file with mtime older than max_staleness considered stale?
+        if sys.platform != 'win32':
+            py.test.skip("max_staleness check only done on win32 because it doesn't support inode change detection")
+
+        self.makefile(self.fname, "foo")
+        uid1 = filesys.fuid(self.fname)
+
+        time.sleep(2) # thanks for waiting :)
+        uid2 = filesys.fuid(self.fname, max_staleness=1)
+        assert uid2 != uid1  # should be considered stale if platform has no inode support
+
+
+coverage_modules = ['MoinMoin.util.filesys']
--- a/MoinMoin/util/filesys.py	Sat May 17 12:23:31 2008 +0200
+++ b/MoinMoin/util/filesys.py	Sat May 17 19:54:30 2008 +0200
@@ -67,6 +67,63 @@
     else:
         os.utime(name, None)
 
+def fuid(filename, max_staleness=3600):
+    """ return a unique id for a file
+
+        Using just the file's mtime to determine if the file has changed is
+        not reliable - if file updates happen faster than the file system's
+        mtime granularity, then the modification is not detectable because
+        the mtime is still the same.
+
+        This function tries to improve by using not only the mtime, but also
+        other metadata values like file size and inode to improve reliability.
+
+        For the calculation of this value, we of course only want to use data
+        that we can get rather fast, thus we use file metadata, not file data
+        (file content).
+
+        Note: depending on the operating system capabilities and the way the
+              file update is done, this function might return the same value
+              even if the file has changed. It should be better than just
+              using file's mtime though.
+              max_staleness tries to avoid the worst for these cases.
+
+        @param filename: file name of the file to look at
+        @param max_staleness: if a file is older than that, we may consider
+                              it stale and return a different uid - this is a
+                              dirty trick to work around changes never being
+                              detected. Default is 3600 seconds, use None to
+                              disable this trickery. See below for more details.
+        @return: an object that changes value if the file changed,
+                 None is returned if there were problems accessing the file
+    """
+    try:
+        st = os.stat(filename)
+    except (IOError, OSError):
+        uid = None  # for permanent errors on stat() this does not change, but
+                    # having a changing value would be pointless because if we
+                    # can't even stat the file, it is unlikely we can read it.
+    else:
+        fake_mtime = int(st.st_mtime)
+        if not st.st_ino and max_staleness:
+            # st_ino being 0 likely means that we run on a platform not
+            # supporting it (e.g. win32) - thus we likely need this dirty
+            # trick
+            now = int(time.time())
+            if now >= st.st_mtime + max_staleness:
+                fake_mtime = now
+        uid = (st.st_mtime,  # might have a rather rough granularity, e.g. 2s
+                             # on FAT and might not change on fast updates
+               st.st_ino,  # inode number (will change if the update is done
+                           # by e.g. renaming a temp file to the real file).
+                           # not supported on win32 (0 ever)
+               st.st_size,  # likely to change on many updates, but not
+                            # sufficient alone
+               fake_mtime,  # trick to workaround file system / platform
+                            # limitations causing permanent trouble
+              )
+    return uid
+
 
 def copystat(src, dst):
     """Copy stat bits from src to dst