changeset 908:a31e0d2f542d storage-ng

reorder processing when storing a revision, fix ITEMLINKS/ITEMTRANSCLUSIONS we need to preprocess data first, e.g. to compute ITEMLINKS/TRANSCLUSIONS. (we also can do some checks here and e.g. add a MTIME if not present) then we need to store to backend (revid is returned) then we need to put stuff into index (needs the revid)
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Fri, 30 Sep 2011 18:14:04 +0200
parents b090aadcc7fb
children a61b3eb8aaa0
files MoinMoin/items/__init__.py MoinMoin/storage/middleware/indexing.py MoinMoin/user.py
diffstat 3 files changed, 21 insertions(+), 12 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/items/__init__.py	Fri Sep 30 17:25:35 2011 +0200
+++ b/MoinMoin/items/__init__.py	Fri Sep 30 18:14:04 2011 +0200
@@ -488,9 +488,6 @@
             # make sure we have CONTENTTYPE
             meta[CONTENTTYPE] = unicode(contenttype_current or contenttype_guessed or 'application/octet-stream')
 
-        if MTIME not in meta:
-            meta[MTIME] = int(time.time())
-
         if ADDRESS not in meta:
             meta[ADDRESS] = u'0.0.0.0' # TODO
 
--- a/MoinMoin/storage/middleware/indexing.py	Fri Sep 30 17:25:35 2011 +0200
+++ b/MoinMoin/storage/middleware/indexing.py	Fri Sep 30 18:14:04 2011 +0200
@@ -179,7 +179,7 @@
         # no way
         raise TypeError("No converter for %s --> %s" % (input_contenttype, output_contenttype))
     except Exception as e: # catch all exceptions, we don't want to break an indexing run
-        logging.exception("Exception happened in conversion of item %r rev %s contenttype %s:" % (meta[NAME], meta[REVID], meta.get(CONTENTTYPE, '')))
+        logging.exception("Exception happened in conversion of item %r rev %s contenttype %s:" % (meta[NAME], meta.get(REVID, 'new'), meta.get(CONTENTTYPE, '')))
         doc = u'ERROR [%s]' % str(e)
         return doc
 
@@ -329,12 +329,14 @@
         self.destroy()
         os.rename(self.index_dir_tmp, self.index_dir)
 
-    def index_revision(self, revid, meta, data, async=True):
+    def index_revision(self, meta, content, async=True):
         """
         Index a single revision, add it to all-revs and latest-revs index.
+
+        :param meta: metadata dict
+        :param content: preprocessed (filtered) indexable content
+        :param async: if True, use the AsyncWriter, otherwise use normal writer
         """
-        meta[REVID] = revid
-        content = convert_to_indexable(meta, data)
         doc = backend_to_index(meta, content, self.schemas[ALL_REVS], self.wikiname)
         if async:
             writer = AsyncWriter(self.ix[ALL_REVS])
@@ -412,7 +414,7 @@
             for revid in revids:
                 if mode in ['add', 'update', ]:
                     meta, data = self.backend.retrieve(revid)
-                    content = convert_to_indexable(meta, data)
+                    content = convert_to_indexable(meta, data, is_new=False)
                     doc = backend_to_index(meta, content, schema, wikiname)
                 if mode == 'update':
                     writer.update_document(**doc)
@@ -754,6 +756,16 @@
         """
         return self[revid]
 
+    def preprocess(self, meta, data):
+        """
+        preprocess a revision before it gets stored and put into index.
+        """
+        meta[ITEMID] = self.itemid
+        if MTIME not in meta:
+            meta[MTIME] = int(time.time())
+        content = convert_to_indexable(meta, data, is_new=True)
+        return meta, data, content
+
     def store_revision(self, meta, data, overwrite=False):
         """
         Store a revision into the backend, write metadata and data to it.
@@ -774,10 +786,11 @@
             revid = meta.get(REVID)
             if revid is not None and revid in backend:
                 raise ValueError('need overwrite=True to overwrite existing revisions')
-        meta[ITEMID] = self.itemid
+        meta, data, content = self.preprocess(meta, data)
+        data.seek(0)  # rewind file
         revid = backend.store(meta, data)
-        data.seek(0)  # rewind file
-        self.indexer.index_revision(revid, meta, data)
+        meta[REVID] = revid
+        self.indexer.index_revision(meta, content)
         if not overwrite:
             self._current = self.indexer._document(all_revs=False, revid=revid)
         return Revision(self, revid)
--- a/MoinMoin/user.py	Fri Sep 30 17:25:35 2011 +0200
+++ b/MoinMoin/user.py	Fri Sep 30 18:14:04 2011 +0200
@@ -368,7 +368,6 @@
             meta[key] = value
         meta[CONTENTTYPE] = u'application/x.moin.userprofile'
         meta[ACTION] = u'SAVE'
-        meta[MTIME] = int(time.time())
         item.store_revision(meta, StringIO(''), overwrite=True)
 
         if not self.disabled: