changeset 472:faeaf423ef69

fix/speedup scandicts for persistent servers imported from: moin--main--1.5--patch-476
author Thomas Waldmann <tw@waldmann-edv.de>
date Tue, 14 Mar 2006 19:49:34 +0000
parents 9b6ba709da88
children c923694dc117
files ChangeLog MoinMoin/wikidicts.py docs/CHANGES
diffstat 3 files changed, 73 insertions(+), 61 deletions(-) [+]
line wrap: on
line diff
--- a/ChangeLog	Mon Mar 13 20:05:27 2006 +0000
+++ b/ChangeLog	Tue Mar 14 19:49:34 2006 +0000
@@ -2,6 +2,20 @@
 # arch-tag: automatic-ChangeLog--arch@arch.thinkmo.de--2003-archives/moin--main--1.5
 #
 
+2006-03-14 20:49:34 GMT	Thomas Waldmann <tw@waldmann-edv.de>	patch-476
+
+    Summary:
+      fix/speedup scandicts for persistent servers
+    Revision:
+      moin--main--1.5--patch-476
+
+    fix/speedup scandicts for persistent servers
+    
+
+    modified files:
+     ChangeLog MoinMoin/wikidicts.py docs/CHANGES
+
+
 2006-03-13 21:05:27 GMT	Thomas Waldmann <tw@waldmann-edv.de>	patch-475
 
     Summary:
--- a/MoinMoin/wikidicts.py	Mon Mar 13 20:05:27 2006 +0000
+++ b/MoinMoin/wikidicts.py	Tue Mar 14 19:49:34 2006 +0000
@@ -294,11 +294,12 @@
     def scandicts(self):
         """scan all pages matching the dict / group regex and init the dictdict"""
         dump = 0
+        request = self.request
 
         # Save now in our internal version format
         now = wikiutil.timestamp2version(int(time.time()))
         try:
-            lastchange = EditLog(self.request).date()
+            lastchange = EditLog(request).date()
         except logfile.LogMissing:
             lastchange = 0
             dump = 1
@@ -309,7 +310,7 @@
             self.__dict__.update(self.cfg.DICTS_DATA)
         except AttributeError:
             try:
-                cache = caching.CacheEntry(self.request, arena, key)
+                cache = caching.CacheEntry(request, arena, key)
                 data = pickle.loads(cache.content())
                 self.__dict__.update(data)
                 
@@ -321,67 +322,62 @@
                 self.reset()
                 dump = 1
 
-        # everything is ok and nothing changed
-        if lastchange < self.namespace_timestamp and dump==0:
-            return
-
-        isdict = re.compile(self.cfg.page_dict_regex, re.UNICODE).search
-        isgroup = re.compile(self.cfg.page_group_regex, re.UNICODE).search
-
-        # check for new groups / dicts from time to time...
-        if now - self.namespace_timestamp >= wikiutil.timestamp2version(60): # 60s
-
-            # Get all pages in the wiki - without user filtering using
-            # filter function - this make the page list about 10 times
-            # faster.
-            dictpages = self.request.rootpage.getPageList(user='', filter=isdict)
-            grouppages = self.request.rootpage.getPageList(user='', filter=isgroup)
+        if lastchange >= self.namespace_timestamp or dump:
+            isdict = re.compile(self.cfg.page_dict_regex, re.UNICODE).search
+            isgroup = re.compile(self.cfg.page_group_regex, re.UNICODE).search
 
-            # remove old entries when dict or group page have been deleted,
-            # add entries when pages have been added
-            # use copies because the dicts are shared via cfg.DICTS_DATA
-            #  and must not be modified
-            olddictdict = self.dictdict.copy()
-            oldgroupdict = self.groupdict.copy()
-            self.dictdict = {}
-            self.groupdict = {}
+            # check for new groups / dicts from time to time...
+            if now - self.namespace_timestamp >= wikiutil.timestamp2version(60): # 60s
+                # Get all pages in the wiki - without user filtering using filter
+                #function - this make the page list about 10 times faster.
+                dictpages = request.rootpage.getPageList(user='', filter=isdict)
+                grouppages = request.rootpage.getPageList(user='', filter=isgroup)
 
-            for pagename in dictpages:
-                if olddictdict.has_key(pagename):
-                    # keep old
-                    self.dictdict[pagename] = olddictdict[pagename]
-                    del olddictdict[pagename]
-                else:
-                    self.adddict(self.request, pagename)
+                # remove old entries when dict or group page have been deleted,
+                # add entries when pages have been added
+                # use copies because the dicts are shared via cfg.DICTS_DATA
+                #  and must not be modified
+                olddictdict = self.dictdict.copy()
+                oldgroupdict = self.groupdict.copy()
+                self.dictdict = {}
+                self.groupdict = {}
+
+                for pagename in dictpages:
+                    if olddictdict.has_key(pagename):
+                        # keep old
+                        self.dictdict[pagename] = olddictdict[pagename]
+                        del olddictdict[pagename]
+                    else:
+                        self.adddict(request, pagename)
+                        dump = 1
+
+                for pagename in grouppages:
+                    if olddictdict.has_key(pagename):
+                        # keep old
+                        self.dictdict[pagename] = olddictdict[pagename]
+                        self.groupdict[pagename] = oldgroupdict[pagename]
+                        del olddictdict[pagename]
+                    else:
+                        self.addgroup(request, pagename)
+                        dump = 1
+
+                if olddictdict: # dict page was deleted
                     dump = 1
 
-            for pagename in grouppages:
-                if olddictdict.has_key(pagename):
-                    # keep old
-                    self.dictdict[pagename] = olddictdict[pagename]
-                    self.groupdict[pagename] = oldgroupdict[pagename]
-                    del olddictdict[pagename]
-                else:
-                    self.addgroup(self.request, pagename)
-                    dump = 1
-
-            if olddictdict: # dict page was deleted
-                dump = 1
+                self.namespace_timestamp = now
 
-            self.namespace_timestamp = now
-
-        # check if groups / dicts have been modified on disk
-        for pagename in self.dictdict.keys():
-            if Page.Page(self.request, pagename).mtime_usecs() >= self.pageupdate_timestamp:
-                if isdict(pagename):
-                    self.adddict(self.request, pagename)
-                elif isgroup(pagename):
-                    self.addgroup(self.request, pagename)
-                dump = 1
-        self.pageupdate_timestamp = now
-        
-        if not self.base_timestamp:
-            self.base_timestamp = int(time.time())
+            # check if groups / dicts have been modified on disk
+            for pagename in self.dictdict.keys():
+                if Page.Page(request, pagename).mtime_usecs() >= self.pageupdate_timestamp:
+                    if isdict(pagename):
+                        self.adddict(request, pagename)
+                    elif isgroup(pagename):
+                        self.addgroup(request, pagename)
+                    dump = 1
+            self.pageupdate_timestamp = now
+            
+            if not self.base_timestamp:
+                self.base_timestamp = int(time.time())
 
         data = {
             "namespace_timestamp": self.namespace_timestamp,
@@ -391,6 +387,7 @@
             "groupdict": self.groupdict,
             "picklever": self.picklever
         }
+        
         if dump:
             # copy unexpanded groups to self.dictdict
             for name, grp in self.groupdict.items():
@@ -399,9 +396,9 @@
             for name in self.groupdict:
                 self.dictdict[name].expandgroups(self)
 
-        cache = caching.CacheEntry(self.request, arena, key)
-        cache.update(pickle.dumps(data, PICKLE_PROTOCOL))
-        
+            cache = caching.CacheEntry(request, arena, key)
+            cache.update(pickle.dumps(data, PICKLE_PROTOCOL))
+            
         # remember it (persistent environments)
         self.cfg.DICTS_DATA = data
 
--- a/docs/CHANGES	Mon Mar 13 20:05:27 2006 +0000
+++ b/docs/CHANGES	Tue Mar 14 19:49:34 2006 +0000
@@ -88,6 +88,7 @@
     * fixed converter error for https:... image urls
     * ThemeBase (use by modern/rightsidebar): removed duplicate AttachFile from
       actions menu (we already have it in editbar)
+    * speedup group/dicts scanning for persistent servers
 
   Other:
     * Added css classes for the rst admonitions. Thanks to TiagoMacambira!