changeset 206:0ff3c43a0a26

storage: iter_items_noindex() does not use the index (use for index rebuild)
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sat, 30 Apr 2011 20:10:44 +0200
parents 57078169052b
children 2652fa715b44
files MoinMoin/storage/__init__.py MoinMoin/storage/backends/acl.py MoinMoin/storage/backends/fileserver.py MoinMoin/storage/backends/flatfile.py MoinMoin/storage/backends/fs.py MoinMoin/storage/backends/fs19.py MoinMoin/storage/backends/fs2.py MoinMoin/storage/backends/hg.py MoinMoin/storage/backends/indexing.py MoinMoin/storage/backends/memory.py MoinMoin/storage/backends/router.py MoinMoin/storage/backends/sqla.py
diffstat 12 files changed, 56 insertions(+), 21 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/storage/__init__.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/__init__.py	Sat Apr 30 20:10:44 2011 +0200
@@ -98,7 +98,6 @@
         """
         # Very simple implementation because we have no indexing
         # or anything like that. If you want to optimize this, override it.
-        # Needs self.iteritems.
         for item in self.iteritems():
             searchterm.prepare()
             if searchterm.evaluate(item):
@@ -148,8 +147,19 @@
 
     def iteritems(self):
         """
-        Returns an iterator over all items available in this backend (like the
-        dict method).
+        Iterate over all items.
+
+        May use an index internally to optimize.
+
+        :rtype: iterator of item objects
+        """
+        raise NotImplementedError()
+
+    def iter_items_noindex(self):
+        """
+        Iterate over all items.
+
+        Must not use an index as this method is used to *build* the index.
 
         :rtype: iterator of item objects
         """
--- a/MoinMoin/storage/backends/acl.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/acl.py	Sat Apr 30 20:10:44 2011 +0200
@@ -136,14 +136,16 @@
         wrapped_item = AclWrapperItem(real_item, self)
         return wrapped_item
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         """
-        @see: Backend.iteritems.__doc__
+        @see: Backend.iter_items_noindex.__doc__
         """
         for item in self.backend.iteritems():
             if self._may(item.name, READ):
                 yield AclWrapperItem(item, self)
 
+    iteritems = iter_items_noindex
+
     def history(self, reverse=True):
         """
         @see: Backend.history.__doc__
--- a/MoinMoin/storage/backends/fileserver.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/fileserver.py	Sat Apr 30 20:10:44 2011 +0200
@@ -63,7 +63,7 @@
         assert path.startswith(root)
         return path[len(root)+1:]
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         for dirpath, dirnames, filenames in os.walk(self.root_dir):
             yield DirItem(self, self._path2item(dirpath))
             for filename in filenames:
@@ -74,6 +74,8 @@
                 else:
                     yield item
 
+    iteritems = iter_items_noindex
+
     def get_item(self, itemname):
         try:
             return FileItem(self, itemname)
--- a/MoinMoin/storage/backends/flatfile.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/flatfile.py	Sat Apr 30 20:10:44 2011 +0200
@@ -85,11 +85,13 @@
             raise ItemAlreadyExistsError("An Item with the name %r already exists!" % (itemname))
         return Item(self, itemname)
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         filenames = os.listdir(self._path)
         for filename in filenames:
             yield Item(self, self._unquote(filename))
 
+    iteritems = iter_items_noindex
+
     def _get_revision(self, item, revno):
         if revno > 0:
             raise NoSuchRevisionError("No Revision #%d on Item %s" % (revno, item.name))
--- a/MoinMoin/storage/backends/fs.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/fs.py	Sat Apr 30 20:10:44 2011 +0200
@@ -125,7 +125,7 @@
 
         return item
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         c = cdb.init(self._name_db)
         r = c.each()
         while r:
@@ -134,6 +134,8 @@
             yield item
             r = c.each()
 
+    iteritems = iter_items_noindex
+
     def _get_revision(self, item, revno):
         item_id = item._fs_item_id
 
--- a/MoinMoin/storage/backends/fs19.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/fs19.py	Sat Apr 30 20:10:44 2011 +0200
@@ -222,7 +222,7 @@
     def has_item(self, itemname):
         return os.path.isfile(self._current_path(itemname))
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         pages_dir = os.path.join(self._path, 'pages')
         for f in os.listdir(pages_dir):
             itemname = unquoteWikiname(f)
@@ -235,6 +235,8 @@
                 for attachitem in item.iter_attachments():
                     yield attachitem
 
+    iteritems = iter_items_noindex
+
     def get_item(self, itemname):
         try:
             # first try to get a page:
@@ -628,7 +630,7 @@
     def has_item(self, itemname):
         return os.path.isfile(self._get_item_path(itemname))
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         for old_id in os.listdir(self._path):
             try:
                 item = FsUserItem(self, old_id=old_id)
@@ -637,6 +639,8 @@
             else:
                 yield item
 
+    iteritems = iter_items_noindex
+
     def get_item(self, itemname):
         return FsUserItem(self, itemname)
 
--- a/MoinMoin/storage/backends/fs2.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/fs2.py	Sat Apr 30 20:10:44 2011 +0200
@@ -189,7 +189,7 @@
 
         return Item(self, itemname, _fs_metadata={})
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         name2id = self._name2id
         results = name2id.select().execute()
         for row in results:
@@ -200,6 +200,8 @@
             yield item
         results.close()
 
+    iteritems = iter_items_noindex
+
     def _get_revision(self, item, revno):
         return StoredRevision(item, revno)
 
--- a/MoinMoin/storage/backends/hg.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/hg.py	Sat Apr 30 20:10:44 2011 +0200
@@ -136,7 +136,7 @@
         item._id = None
         return item
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         """
         Return generator for iterating through collection of Items
         in repository.
@@ -157,6 +157,8 @@
             yield item
             record = c.each()
 
+    iteritems = iter_items_noindex
+
     def history(self, reverse=True):
         """
         Return generator for iterating in given direction over Item Revisions
--- a/MoinMoin/storage/backends/indexing.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/indexing.py	Sat Apr 30 20:10:44 2011 +0200
@@ -257,7 +257,7 @@
     def index_rebuild(self, backend):
         self.metadata.drop_all()
         self.metadata.create_all()
-        for item in backend.iteritems():
+        for item in backend.iter_items_noindex():
             item.update_index()
             for revno in item.list_revisions():
                 rev = item.get_revision(revno)
--- a/MoinMoin/storage/backends/memory.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/memory.py	Sat Apr 30 20:10:44 2011 +0200
@@ -144,13 +144,15 @@
         rev_hist = [rev for rev in self._revision_history if rev.item.name != item.name]
         self._revision_history = rev_hist
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         """
-        @see: Backend.iteritems.__doc__
+        @see: Backend.iter_items_noindex.__doc__
         """
         for itemname in self._itemmap.keys():
             yield self.get_item(itemname)
 
+    iteritems = iter_items_noindex
+
     def _get_revision(self, item, revno):
         """
         @see: Backend._get_revision.__doc__
--- a/MoinMoin/storage/backends/router.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/router.py	Sat Apr 30 20:10:44 2011 +0200
@@ -96,16 +96,21 @@
         """
         return self._get_backend(namespace)[0]
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         """
-        Iterate over all items. Necessary for traversal.
+        Iterate over all items.
 
-        @see: Backend.iteritems.__doc__
+        Must not use the index as this method is used to *build* the index.
+
+        @see: Backend.iter_items_noindex.__doc__
         """
         for mountpoint, backend in self.mapping:
-            for item in backend.iteritems():
+            for item in backend.iter_items_noindex():
                 yield RouterItem(self, item.name, item, mountpoint)
 
+    # TODO: implement a faster iteritems using the index
+    iteritems = iter_items_noindex
+
     def has_item(self, itemname):
         """
         @see: Backend.has_item.__doc__
--- a/MoinMoin/storage/backends/sqla.py	Sat Apr 30 18:54:05 2011 +0200
+++ b/MoinMoin/storage/backends/sqla.py	Sat Apr 30 20:10:44 2011 +0200
@@ -191,7 +191,7 @@
             yield rev
         session.close()
 
-    def iteritems(self):
+    def iter_items_noindex(self):
         """
         Returns an iterator over all items available in this backend.
         (Like the dict method).
@@ -201,7 +201,7 @@
         The benefit is that we do not issue a query for each individual item, but
         only a single query.
 
-        @see: Backend.history.__doc__
+        @see: Backend.iter_items_noindex.__doc__
         """
         session = self.Session()
         all_items = session.query(SQLAItem).all()
@@ -210,6 +210,8 @@
             item.setup(self)
             yield item
 
+    iteritems = iter_items_noindex
+
     def _create_revision(self, item, revno):
         """
         @see: Backend._create_revision.__doc__