changeset 814:a4ab4cb7614f

merged pytest2 branch into default branch
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Thu, 01 Sep 2011 00:16:33 +0200
parents e00c0eeb6e55 (current diff) 66fee7ba0264 (diff)
children cd3b797c8219
files MoinMoin/apps/frontend/views.py MoinMoin/script/maint/index.py MoinMoin/search/indexing.py MoinMoin/storage/_tests/test_indexing.py MoinMoin/storage/_tests/test_terms.py MoinMoin/storage/backends/indexing.py MoinMoin/storage/terms.py MoinMoin/templates/search_results.html MoinMoin/templates/wanteds.html docs/index.rst
diffstat 128 files changed, 3175 insertions(+), 2181 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/_tests/ldap_testbase.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/_tests/ldap_testbase.py	Thu Sep 01 00:16:33 2011 +0200
@@ -2,7 +2,7 @@
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-    LDAPTestBase: LDAP testing support for py.test based unit tests
+    LDAPTestBase: LDAP testing support for pytest based unit tests
 
     Features
     --------
@@ -226,10 +226,10 @@
         shutil.rmtree(self.ldap_dir)
 
 try:
-    import py.test
+    import pytest
 
     class LDAPTstBase:
-        """ Test base class for py.test based tests which need a LDAP server to talk to.
+        """ Test base class for pytest based tests which need a LDAP server to talk to.
 
             Inherit your test class from this base class to test LDAP stuff.
         """
@@ -247,7 +247,7 @@
             self.ldap_env.create_env(slapd_config=self.slapd_config)
             started = self.ldap_env.start_slapd()
             if not started:
-                py.test.skip("Failed to start %s process, please see your syslog / log files"
+                pytest.skip("Failed to start %s process, please see your syslog / log files"
                              " (and check if stopping apparmor helps, in case you use it)." % SLAPD_EXECUTABLE)
             self.ldap_env.load_directory(ldif_content=self.ldif_content)
 
@@ -257,5 +257,5 @@
             self.ldap_env.destroy_env()
 
 except ImportError:
-    pass  # obviously py.test not in use
+    pass  # obviously pytest not in use
 
--- a/MoinMoin/_tests/test_error.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/_tests/test_error.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,7 +8,7 @@
 """
 
 
-import py
+import pytest
 
 from MoinMoin import error
 
@@ -46,5 +46,31 @@
         err = error.Error(test)
         assert '%(message)s' % err == test
 
+class TestCompositeError(object):
+
+    def setup_method(self, method):
+        self.CompositeError_obj = error.CompositeError(error.InternalError)
+
+    def teardown_method(self, method):
+        self.CompositeError_obj.innerException = None
+
+    def test_exceptions(self):
+        self.CompositeError_obj.innerException = 'test_error1', 'test_error2', 'test_error3'
+        result = error.CompositeError.exceptions(self.CompositeError_obj)
+        expected = [('test_error1', 'test_error2', 'test_error3')]
+        assert expected == result
+        self.CompositeError_obj.innerException = str(error.InternalError(''))
+
+    def test_subclasses(self):
+        self.CompositeError_obj.innerException = str(error.FatalError('This is an internal Error'))
+        result = error.CompositeError.exceptions(self.CompositeError_obj)
+        expected = ['This is an internal Error']
+        assert result == expected
+
+        self.CompositeError_obj.innerException = str(error.FatalError('This is a fatal Error'))
+        result = error.CompositeError.exceptions(self.CompositeError_obj)
+        expected = ['This is a fatal Error']
+        assert result == expected
+
 coverage_modules = ['MoinMoin.error']
 
--- a/MoinMoin/_tests/test_sourcecode.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/_tests/test_sourcecode.py	Thu Sep 01 00:16:33 2011 +0200
@@ -10,9 +10,10 @@
 
 
 import re, time
-import py
+import pytest
 from . import pep8
 
+import py
 
 moindir = py.path.local(__file__).pypkgpath()
 
@@ -100,9 +101,9 @@
 def test_sourcecode(path):
     mtime = path.stat().mtime
     if mtime < RECENTLY:
-        py.test.skip("source change not recent")
+        pytest.skip("source change not recent")
     if not should_check_file(str(path), mtime):
-        py.test.skip("source marked as should not be checked")
+        pytest.skip("source marked as should not be checked")
 
     check_py_file(str(moindir.bestrelpath(path)), str(path), mtime)
 
--- a/MoinMoin/_tests/test_test_environ.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/_tests/test_test_environ.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,19 +6,22 @@
 """
 
 
-import py
+import pytest
 
 from flask import current_app as app
 from flask import g as flaskg
 
+from MoinMoin.conftest import init_test_app, deinit_test_app
 from MoinMoin.config import NAME, CONTENTTYPE, IS_SYSITEM, SYSITEM_VERSION
 from MoinMoin.storage.error import NoSuchItemError
+from MoinMoin.storage.serialization import serialize, unserialize
 
 from MoinMoin._tests import wikiconfig
 
 class TestStorageEnvironWithoutConfig(object):
     def setup_method(self, method):
         self.class_level_value = 123
+        app, ctx = init_test_app(wikiconfig.Config)
 
     def test_fresh_backends(self):
         assert self.class_level_value == 123
@@ -28,11 +31,9 @@
         storage = flaskg.storage
         assert storage
         assert hasattr(storage, 'get_item')
-        assert hasattr(storage, 'history')
         assert not list(storage.iteritems())
-        assert not list(storage.history())
         itemname = u"this item shouldn't exist yet"
-        assert py.test.raises(NoSuchItemError, storage.get_item, itemname)
+        assert pytest.raises(NoSuchItemError, storage.get_item, itemname)
         item = storage.create_item(itemname)
         new_rev = item.create_revision(0)
         new_rev[NAME] = itemname
@@ -44,9 +45,9 @@
     # Run this test twice to see if something's changed
     test_twice = test_fresh_backends
 
+
 class TestStorageEnvironWithConfig(object):
     class Config(wikiconfig.Config):
-        load_xml = wikiconfig.Config._test_items_xml
         content_acl = dict(
             before="+All:write", # need to write to sys pages
             default="All:read,write,admin,create,destroy",
@@ -55,6 +56,10 @@
         )
 
     def test_fresh_backends_with_content(self):
+        # get the items from xml file
+        backend = app.unprotected_storage
+        unserialize(backend, self.Config._test_items_xml)
+
         assert isinstance(app.cfg, wikiconfig.Config)
 
         storage = flaskg.storage
--- a/MoinMoin/_tests/test_user.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/_tests/test_user.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,7 +8,7 @@
 """
 
 
-import py
+import pytest
 
 from flask import current_app as app
 from flask import g as flaskg
@@ -121,7 +121,7 @@
             theuser = user.User(name=name, password='12345')
             assert theuser.valid
         except ImportError:
-            py.test.skip("Platform does not provide crypt module!")
+            pytest.skip("Platform does not provide crypt module!")
 
     def test_auth_with_ssha256_stored_password(self):
         """
@@ -299,6 +299,118 @@
         theuser = user.User(uid)
         assert theuser.email == email
 
+    # Bookmarks -------------------------------------------------------
+
+    def test_bookmark(self):
+        name = u'Test_User_quicklink'
+        password = name
+        self.createUser(name, password)
+        theUser = user.User(name=name, password=password)
+
+        theUser.setBookmark(7)
+        result_added = theUser.getBookmark()
+        expected = 7
+        assert result_added == expected
+        # delete the bookmark
+        result_success = theUser.delBookmark()
+        assert result_success == 0
+        result_deleted = theUser.getBookmark()
+        assert not result_deleted
+
+        # delBookmark should return 1 on failure
+        result_failure = theUser.delBookmark()
+        assert result_failure == 1
+
+    # Quicklinks ------------------------------------------------------
+
+    def test_quicklinks(self):
+        """
+        Test for the quicklinks
+        """
+        pagename = u'Test_page_quicklink'
+        name = u'Test_User_quicklink'
+        password = name
+        self.createUser(name, password)
+        theUser = user.User(name=name, password=password)
+        theUser.subscribe(pagename)
+
+        # no quick links exist yet
+        result_before = theUser.getQuickLinks()
+        assert result_before == []
+
+        result = theUser.isQuickLinkedTo([pagename])
+        assert not result
+
+        # quicklinks for the user - theUser exist now
+        theUser.quicklinks = [pagename]
+        result_after = theUser.getQuickLinks()
+        expected = [u'Test_page_quicklink']
+        assert result_after == expected
+
+        # test for addQuicklink()
+        theUser.addQuicklink(u'Test_page_added')
+        result_on_addition = theUser.getQuickLinks()
+        expected = [u'Test_page_quicklink', u'MoinTest:Test_page_added']
+        assert result_on_addition == expected
+
+        # user should be quicklinked to [pagename]
+        result = theUser.isQuickLinkedTo([pagename])
+        assert result
+
+        # previously added page u'Test_page_added' is removed
+        theUser.removeQuicklink(u'Test_page_added')
+        result_on_removal = theUser.getQuickLinks()
+        expected = [u'Test_page_quicklink']
+        assert result_on_removal == expected
+
+    # Trail -----------------------------------------------------------
+
+    def test_trail(self):
+        pagename = u'Test_page_trail'
+        name = u'Test_User_trail'
+        password = name
+        self.createUser(name, password)
+        theUser = user.User(name=name, password=password)
+
+        # no item name added to trail
+        result = theUser.getTrail()
+        expected = []
+        assert result == expected
+
+        # item name added to trail
+        theUser.addTrail(u'item_added')
+        result = theUser.getTrail()
+        expected = [u'MoinTest:item_added']
+        assert result == expected
+
+    # Other ----------------------------------------------------------
+
+    def test_signature(self):
+        name = u'Test_User_other'
+        password = name
+        self.createUser(name, password)
+        theUser = user.User(name=name, password=password)
+
+        # test the user signature
+        result = theUser.signature()
+        expected =  u'[[Test_User_other]]'
+        assert result == expected
+
+    def test_recovery_token(self):
+        name = u'Test_User_other'
+        password = name
+        self.createUser(name, password)
+        theUser = user.User(name=name, password=password)
+
+        # use recovery token to generate new password
+        test_token = theUser.generate_recovery_token()
+        result_success = theUser.apply_recovery_token(test_token, u'test_newpass')
+        assert result_success
+
+        # wrong token
+        result_failure = theUser.apply_recovery_token('test_wrong_token', u'test_newpass')
+        assert not result_failure
+
     # Helpers ---------------------------------------------------------
 
     def createUser(self, name, password, pwencoded=False, email=None):
@@ -315,7 +427,7 @@
         # Validate that we are not modifying existing user data file!
         if self.user.exists():
             self.user = None
-            py.test.skip("Test user exists, will not override existing user data file!")
+            pytest.skip("Test user exists, will not override existing user data file!")
 
         # Save test user
         self.user.save()
@@ -323,7 +435,7 @@
         # Validate user creation
         if not self.user.exists():
             self.user = None
-            py.test.skip("Can't create test user")
+            pytest.skip("Can't create test user")
 
 
 class TestGroupName(object):
--- a/MoinMoin/_tests/test_wikiutil.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/_tests/test_wikiutil.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,12 +7,13 @@
 """
 
 
-import py
+import pytest
 
 from flask import current_app as app
 
 from MoinMoin import config, wikiutil
 from MoinMoin._tests import wikiconfig
+from MoinMoin.storage.serialization import serialize, unserialize
 
 from werkzeug import MultiDict
 
@@ -37,11 +38,12 @@
         'NoSuchPageYetAndWillNeverBe',
         )
 
-    class Config(wikiconfig.Config):
-        load_xml = wikiconfig.Config._test_items_xml
-
     def testSystemItem(self):
         """wikiutil: good system item names accepted, bad rejected"""
+        # get the items from xml file
+        backend = app.unprotected_storage
+        unserialize(backend, wikiconfig.Config._test_items_xml)
+
         for name in self.systemItems:
             assert wikiutil.isSystemItem(name)
         for name in self.notSystemItems:
@@ -175,5 +177,92 @@
                 assert result == expected
 
 
+def testParentItemName():
+    # with no parent
+    result = wikiutil.ParentItemName(u'itemname')
+    expected = u''
+    assert result == expected, ('Expected "%(expected)s" but got "%(result)s"')
+    # with a parent
+    result = wikiutil.ParentItemName(u'some/parent/itemname')
+    expected = u'some/parent'
+    assert result == expected
+
+def testdrawing2fname():
+    # with extension not in config.drawing_extensions
+    result = wikiutil.drawing2fname('Moin_drawing.txt')
+    expected = 'Moin_drawing.txt.tdraw'
+    assert result == expected
+    # with extension in config.drawing_extensions
+    result = wikiutil.drawing2fname('Moindir.Moin_drawing.jpg')
+    expected = 'Moindir.Moin_drawing.jpg'
+    assert result == expected
+
+def testgetUnicodeIndexGroup():
+    result = wikiutil.getUnicodeIndexGroup(['moin-2', 'MoinMoin'])
+    expected = 'MOIN-2'
+    assert result == expected
+    # empty char
+    with pytest.raises(IndexError):
+        result = wikiutil.getUnicodeIndexGroup('')
+
+def testis_URL():
+    sample_schemas = ['http', 'https', 'ftp', 'ssh']
+    for schema in sample_schemas:
+        result = wikiutil.is_URL(schema + ':MoinMoin')
+        assert result
+
+    # arg without ':' which is a mandatory requirement
+    result = wikiutil.is_URL('MoinMoin')
+    assert not result
+    # invalid schema
+    result = wikiutil.is_URL('invalid_schema:MoinMoin')
+    assert not result
+
+def testcontainsConflictMarker():
+    # text with conflict marker
+    result = wikiutil.containsConflictMarker("/!\\ '''Edit conflict - Conflict marker is present")
+    assert result
+
+    #text without conflict marker
+    result = wikiutil.containsConflictMarker('No conflict marker')
+    assert not result
+
+def testsplit_anchor():
+    """
+    TODO: add the test for for split_anchor when we have better
+          approach to deal wih problems like "#MoinMoin#" returning ("#MoinMoin", "")
+    """
+    result = wikiutil.split_anchor('MoinMoin')
+    expected = 'MoinMoin', ''
+    assert result == expected
+
+    result = wikiutil.split_anchor('MoinMoin#test_anchor|label|attr=val')
+    expected = ['MoinMoin', 'test_anchor|label|attr=val']
+    assert result == expected
+
+    result = wikiutil.split_anchor('#MoinMoin#')
+    expected = ['#MoinMoin', '']
+    assert result == expected
+
+def testfile_headers():
+    test_headers = [
+                #test_file               #content_type
+                ('imagefile.gif',       'image/gif'),
+                ('testfile.txt',        'text/plain; charset="utf-8"'),
+                ('pdffile.pdf',         'application/pdf'),
+                ('docfile.doc',         'application/msword'),
+                (None,                  'application/octet-stream')
+                ]
+
+    for test_file, content_type in test_headers:
+        result = wikiutil.file_headers(test_file, None, 10)
+        expected = [('Content-Type', content_type), ('Content-Length', '10')]
+        assert result == expected
+
+    # filename is none and content type has a value
+    result = wikiutil.file_headers(None, 'plane/text')
+    expected = [('Content-Type', 'plane/text')]
+    assert result == expected
+
 coverage_modules = ['MoinMoin.wikiutil']
 
--- a/MoinMoin/app.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/app.py	Thu Sep 01 00:16:33 2011 +0200
@@ -133,9 +133,6 @@
     clock.start('create_app init backends')
     app.unprotected_storage, app.storage = init_backends(app)
     clock.stop('create_app init backends')
-    clock.start('create_app load/save xml')
-    import_export_xml(app)
-    clock.stop('create_app load/save xml')
     clock.start('create_app flask-babel')
     i18n_init(app)
     clock.stop('create_app flask-babel')
@@ -184,47 +181,6 @@
     app.unprotected_storage.close()
 
 
-def import_export_xml(app):
-    # If the content was already pumped into the backend, we don't want
-    # to do that again. (Works only until the server is restarted.)
-    xmlfile = app.cfg.load_xml
-    if xmlfile:
-        app.cfg.load_xml = None
-        tmp_backend = router.RouterBackend([('/', memory.MemoryBackend())], cfg=app.cfg)
-        unserialize(tmp_backend, xmlfile)
-        # TODO optimize this, maybe unserialize could count items it processed
-        item_count = 0
-        for item in tmp_backend.iteritems():
-            item_count += 1
-        logging.debug("loaded xml into tmp_backend: %s, %d items" % (xmlfile, item_count))
-        try:
-            # In case the server was restarted we cannot know whether
-            # the xml data already exists in the target backend.
-            # Hence we check the existence of the items before we unserialize
-            # them to the backend.
-            backend = app.unprotected_storage
-            for item in tmp_backend.iteritems():
-                item = backend.get_item(item.name)
-        except StorageError:
-            # if there is some exception, we assume that backend needs to be filled
-            # we need to use it as unserialization target so that update mode of
-            # unserialization creates the correct item revisions
-            logging.debug("unserialize xml file %s into %r" % (xmlfile, backend))
-            unserialize(backend, xmlfile)
-    else:
-        item_count = 0
-
-    # XXX wrong place / name - this is a generic preload functionality, not just for tests
-    # To make some tests happy
-    app.cfg.test_num_pages = item_count
-
-    xmlfile = app.cfg.save_xml
-    if xmlfile:
-        app.cfg.save_xml = None
-        backend = app.unprotected_storage
-        serialize(backend, xmlfile)
-
-
 def setup_user():
     """
     Try to retrieve a valid user object from the request, be it
--- a/MoinMoin/apps/admin/views.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/apps/admin/views.py	Thu Sep 01 00:16:33 2011 +0200
@@ -21,7 +21,7 @@
 from MoinMoin.apps.admin import admin
 from MoinMoin import user
 from MoinMoin.storage.error import NoSuchRevisionError
-from MoinMoin.config import SIZE
+from MoinMoin.config import NAME, SIZE
 from MoinMoin.config import SUPERUSER
 from MoinMoin.security import require_permission
 
@@ -229,15 +229,8 @@
     headings = [_('Size'),
                 _('Item name'),
                ]
-    rows = []
-    for item in flaskg.storage.iteritems():
-        try:
-            rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            # XXX we currently also get user items, they have no revisions -
-            # but in the end, they should not be readable by the user anyways
-            continue
-        rows.append((rev[SIZE], item.name))
+    rows = [(doc[SIZE], doc[NAME])
+            for doc in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)]
     rows = sorted(rows, reverse=True)
     return render_template('admin/itemsize.html',
                            item_name="+admin/itemsize",
--- a/MoinMoin/apps/feed/views.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/apps/feed/views.py	Thu Sep 01 00:16:33 2011 +0200
@@ -17,13 +17,15 @@
 
 from werkzeug.contrib.atom import AtomFeed
 
+from whoosh.query import Term, And
+
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
 from MoinMoin import wikiutil
 from MoinMoin.i18n import _, L_, N_
 from MoinMoin.apps.feed import feed
-from MoinMoin.config import NAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT
+from MoinMoin.config import NAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT, MTIME
 from MoinMoin.themes import get_editor_info
 from MoinMoin.items import Item
 from MoinMoin.util.crypto import cache_key
@@ -43,11 +45,15 @@
     if content is None:
         title = app.cfg.sitename
         feed = AtomFeed(title=title, feed_url=request.url, url=request.host_url)
-        for rev in flaskg.storage.history(item_name=item_name):
-            this_rev = rev
-            this_revno = rev.revno
-            item = rev.item
-            name = rev[NAME]
+        query = Term("wikiname", app.cfg.interwikiname)
+        if item_name:
+            query = And([query, Term("name_exact", item_name), ])
+        history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True, limit=100)
+        for doc in history:
+            name = doc[NAME]
+            this_revno = doc["rev_no"]
+            item = flaskg.storage.get_item(name)
+            this_rev = item.get_revision(this_revno)
             try:
                 hl_item = Item.create(name, rev_no=this_revno)
                 previous_revno = this_revno - 1
@@ -65,11 +71,11 @@
                 content = _(u'MoinMoin feels unhappy.')
                 content_type = 'text'
             feed.add(title=name, title_type='text',
-                     summary=rev.get(COMMENT, ''), summary_type='text',
+                     summary=doc.get(COMMENT, ''), summary_type='text',
                      content=content, content_type=content_type,
-                     author=get_editor_info(rev, external=True),
+                     author=get_editor_info(doc, external=True),
                      url=url_for_item(name, rev=this_revno, _external=True),
-                     updated=datetime.utcfromtimestamp(rev.timestamp),
+                     updated=doc[MTIME],
                     )
         content = feed.to_string()
         app.cache.set(cid, content)
--- a/MoinMoin/apps/frontend/_tests/test_frontend.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/apps/frontend/_tests/test_frontend.py	Thu Sep 01 00:16:33 2011 +0200
@@ -12,6 +12,8 @@
 from MoinMoin.apps.frontend import views
 from MoinMoin import user
 from MoinMoin.util import crypto
+from MoinMoin._tests import wikiconfig
+import pytest
 
 class TestFrontend(object):
     def test_root(self):
@@ -49,6 +51,22 @@
             assert '<html>' in rv.data
             assert '</html>' in rv.data
 
+    def test_wanteds(self):
+        with self.app.test_client() as c:
+            rv = c.get('/+wanteds')
+            assert rv.status == '200 OK'
+            assert rv.headers['Content-Type'] == 'text/html; charset=utf-8'
+            assert '<html>' in rv.data
+            assert '</html>' in rv.data
+
+    def test_orphans(self):
+        with self.app.test_client() as c:
+            rv = c.get('/+orphans')
+            assert rv.status == '200 OK'
+            assert rv.headers['Content-Type'] == 'text/html; charset=utf-8'
+            assert '<html>' in rv.data
+            assert '</html>' in rv.data
+
 class TestUsersettings(object):
     def setup_method(self, method):
         # Save original user
@@ -146,7 +164,7 @@
         # Validate that we are not modifying existing user data file!
         if self.user.exists():
             self.user = None
-            py.test.skip("Test user exists, will not override existing user data file!")
+            pytest.skip("Test user exists, will not override existing user data file!")
 
         # Save test user
         self.user.save()
@@ -154,68 +172,4 @@
         # Validate user creation
         if not self.user.exists():
             self.user = None
-            py.test.skip("Can't create test user")
-
-
-class TestViews(object):
-    """
-    Tester class for +backrefs, +orphans and +wanted views
-    """
-    class DummyItem(object):
-        """
-        Fake storage object, simulating the page item object from the storage
-        """
-        def __init__(self, name, revision):
-            self.latest_revision = revision
-            self.name = name
-
-        def get_revision(self, *args, **kw):
-            return self.latest_revision
-
-    class DummyRevision(object):
-        """
-        Fake revision object, used for retrieving ITEMTRANSCLUSIONS and ITEMLINKS meta
-        """
-        def __init__(self, links, transclusions):
-            self.links = links
-            self.transclusions = transclusions
-
-        def get(self, meta_name, *args, **kw):
-            if meta_name == 'itemlinks':
-                return self.links
-            if meta_name == 'itemtransclusions':
-                return self.transclusions
-
-    def setup_class(self):
-        # list of tuples
-        # (page_name, links, transclusions)
-        items = [('page1', ['page2', 'page3'], ['page2']),
-                 ('page2',  ['page1', 'page3'], []),
-                 ('page3', ['page5'], ['page1']),
-                 ('page4', [], ['page5'])
-                ]
-        # we create the list of items
-        self.items = []
-        for item in items:
-            revision = self.DummyRevision(item[1], item[2])
-            page = self.DummyItem(item[0], revision)
-            self.items.append(page)
-
-    def test_orphans(self):
-        expected_orphans = sorted(['page4'])
-        result_orphans = sorted(views._orphans(self.items))
-
-        assert result_orphans == expected_orphans
-
-    def test_wanteds(self):
-        expected_wanteds = {'page5': ['page3', 'page4']}
-        result_wanteds = views._wanteds(self.items)
-
-        assert result_wanteds == expected_wanteds
-
-    def test_backrefs(self):
-        expected_backrefs = sorted(['page1', 'page2'])
-        result_backrefs = sorted(views._backrefs(self.items, 'page3'))
-
-        assert result_backrefs == expected_backrefs
-
+            pytest.skip("Can't create test user")
--- a/MoinMoin/apps/frontend/views.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/apps/frontend/views.py	Thu Sep 01 00:16:33 2011 +0200
@@ -39,6 +39,8 @@
 import pytz
 from babel import Locale
 
+from whoosh.query import Term, And, Or, DateRange
+
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
@@ -48,7 +50,7 @@
 from MoinMoin.items import Item, NonExistent
 from MoinMoin.items import ROWS_META, COLS, ROWS_DATA
 from MoinMoin import config, user, util, wikiutil
-from MoinMoin.config import ACTION, COMMENT, CONTENTTYPE, ITEMLINKS, ITEMTRANSCLUSIONS, NAME, CONTENTTYPE_GROUPS
+from MoinMoin.config import ACTION, COMMENT, CONTENTTYPE, ITEMLINKS, ITEMTRANSCLUSIONS, NAME, CONTENTTYPE_GROUPS, MTIME, TAGS
 from MoinMoin.util import crypto
 from MoinMoin.util.interwiki import url_for_item
 from MoinMoin.security.textcha import TextCha, TextChaizedForm, TextChaValid
@@ -107,6 +109,7 @@
 Disallow: /+diffsince/
 Disallow: /+diff/
 Disallow: /+diffraw/
+Disallow: /+search
 Disallow: /+dispatch/
 Disallow: /+admin/
 Allow: /
@@ -134,46 +137,45 @@
         return True
 
 class SearchForm(Form):
-    q = String.using(optional=False).with_properties(autofocus=True, placeholder=L_("Search Query"))
+    q = String.using(optional=False, default=u'').with_properties(autofocus=True, placeholder=L_("Search Query"))
+    history = Boolean.using(label=L_('search also in non-current revisions'), optional=True)
     submit = String.using(default=L_('Search'), optional=True)
-    pagelen = String.using(optional=False)
-    search_in_all = Boolean.using(label=L_('search also in non-current revisions'), optional=True)
 
     validators = [ValidSearch()]
 
 
-def _search(search_form, item_name):
-    from MoinMoin.search.indexing import WhooshIndex
-    from whoosh.qparser import QueryParser, MultifieldParser
-    from MoinMoin.search.analyzers import item_name_analyzer
-    from whoosh import highlight
+@frontend.route('/+search', methods=['GET', 'POST'])
+def search():
+    search_form = SearchForm.from_flat(request.values)
+    valid = search_form.validate()
+    search_form['submit'].set_default() # XXX from_flat() kills all values
     query = search_form['q'].value
-    pagenum = 1 # We start from first page
-    pagelen = search_form['pagelen'].value
-    index_object = WhooshIndex()
-    ix = index_object.all_revisions_index if request.values.get('search_in_all') else index_object.latest_revisions_index
-    with ix.searcher() as searcher:
-        mparser = MultifieldParser(["name_exact", "name", "content"], schema=ix.schema)
-        q = mparser.parse(query)
-        results = searcher.search_page(q, int(pagenum), pagelen=int(pagelen))
-        return render_template('search_results.html',
-                               results=results,
+    if valid:
+        history = bool(request.values.get('history'))
+        qp = flaskg.storage.query_parser(["name_exact", "name", "content"], all_revs=history)
+        q = qp.parse(query)
+        with flaskg.storage.searcher(all_revs=history) as searcher:
+            results = searcher.search(q, limit=100)
+            return render_template('search.html',
+                                   results=results,
+                                   name_suggestions=u', '.join([word for word, score in results.key_terms('name', docs=20, numterms=10)]),
+                                   content_suggestions=u', '.join([word for word, score in results.key_terms('content', docs=20, numterms=10)]),
+                                   query=query,
+                                   medium_search_form=search_form,
+                                   item_name='+search', # XXX
+                                  )
+    else:
+        return render_template('search.html',
                                query=query,
                                medium_search_form=search_form,
-                               item_name=item_name,
+                               item_name='+search', # XXX
                               )
 
 
 
-@frontend.route('/<itemname:item_name>', defaults=dict(rev=-1), methods=['GET', 'POST'])
-@frontend.route('/+show/<int:rev>/<itemname:item_name>', methods=['GET', 'POST'])
+@frontend.route('/<itemname:item_name>', defaults=dict(rev=-1), methods=['GET'])
+@frontend.route('/+show/<int:rev>/<itemname:item_name>', methods=['GET'])
 def show_item(item_name, rev):
-    # first check whether we have a valid search query:
-    search_form = SearchForm.from_flat(request.values)
-    if search_form.validate():
-        return _search(search_form, item_name)
-    search_form['submit'].set_default() # XXX from_flat() kills all values
-
     flaskg.user.addTrail(item_name)
     item_displayed.send(app._get_current_object(),
                         item_name=item_name)
@@ -203,7 +205,7 @@
                               data_rendered=Markup(item._render_data()),
                               show_revision=show_revision,
                               show_navigation=show_navigation,
-                              search_form=search_form,
+                              search_form=SearchForm.from_defaults(),
                              )
     return Response(content, status)
 
@@ -668,7 +670,7 @@
     :type item_name: unicode
     :returns: a page with all the items which link or transclude item_name
     """
-    refs_here = _backrefs(flaskg.storage.iteritems(), item_name)
+    refs_here = _backrefs(item_name)
     return render_template('item_link_list.html',
                            item_name=item_name,
                            headline=_(u'Refers Here'),
@@ -676,96 +678,104 @@
                           )
 
 
-def _backrefs(items, item_name):
+def _backrefs(item_name):
     """
     Returns a list with all names of items which ref item_name
 
-    :param items: all the items
-    :type items: iteratable sequence
     :param item_name: the name of the item transcluded or linked
     :type item_name: unicode
     :returns: the list of all items which ref item_name
     """
-    from MoinMoin.search.indexing import WhooshIndex
-    from whoosh.query import Term, Or
-    index_object = WhooshIndex()
-    ix = index_object.latest_revisions_index
-    with ix.searcher() as searcher:
-        q = Or([Term("itemtransclusions", item_name), Term("itemlinks", item_name)])
-        results = searcher.search(q)
-        return [result["name"] for result in results]
+    q = And([Term("wikiname", app.cfg.interwikiname),
+             Or([Term("itemtransclusions", item_name), Term("itemlinks", item_name)])])
+    docs = flaskg.storage.search(q, all_revs=False)
+    return [doc["name"] for doc in docs]
+
 
 @frontend.route('/+history/<itemname:item_name>')
 def history(item_name):
-    history = flaskg.storage.history(item_name=item_name)
-
     offset = request.values.get('offset', 0)
     offset = max(int(offset), 0)
-
-    results_per_page = int(app.cfg.results_per_page)
     if flaskg.user.valid:
         results_per_page = flaskg.user.results_per_page
+    else:
+        results_per_page = app.cfg.results_per_page
+    query = And([Term("wikiname", app.cfg.interwikiname), Term("name_exact", item_name), ])
+    # TODO: due to how getPageContent and the template works, we need to use limit=None -
+    # it would be better to use search_page (and an appropriate limit, if needed)
+    docs = flaskg.storage.search(query, all_revs=True, sortedby="rev_no", reverse=True, limit=None)
+    # get rid of the content value to save potentially big amounts of memory:
+    history = [dict((k, v) for k, v in doc.iteritems() if k != 'content') for doc in docs]
     history_page = util.getPageContent(history, offset, results_per_page)
-
     return render_template('history.html',
                            item_name=item_name, # XXX no item here
                            history_page=history_page,
                           )
 
+
 @frontend.route('/+history')
 def global_history():
-    history = flaskg.storage.history(item_name='')
-    results_per_page = int(app.cfg.results_per_page)
+    bookmark_time = None
     if flaskg.user.valid:
-        bookmark_time = flaskg.user.getBookmark()
-        results_per_page = flaskg.user.results_per_page # if it is 0, means no paging
+        bm = flaskg.user.getBookmark()
+        if bm is not None:
+            bookmark_time = datetime.utcfromtimestamp(bm)
+    if flaskg.user.valid:
+        results_per_page = flaskg.user.results_per_page
     else:
-        bookmark_time = None
+        results_per_page = app.cfg.results_per_page
+    query = Term("wikiname", app.cfg.interwikiname)
+    if bookmark_time is not None:
+        query = And([query, DateRange(MTIME, start=bookmark_time, end=None)])
+    # TODO: we need use limit=None to simulate previous implementation's behaviour -
+    # it would be better to use search_page (and an appropriate limit, if needed)
+    history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True, limit=None)
     item_groups = OrderedDict()
-    for rev in history:
-        current_item_name = rev.item.name
-        if bookmark_time and rev.timestamp <= bookmark_time:
+    for doc in history:
+        current_item_name = doc[NAME]
+        if bookmark_time and doc[MTIME] <= bookmark_time:
             break
         elif current_item_name in item_groups:
-            latest_rev = item_groups[current_item_name][0]
-            tm_latest = datetime.utcfromtimestamp(latest_rev.timestamp)
-            tm_current = datetime.utcfromtimestamp(rev.timestamp)
+            latest_doc = item_groups[current_item_name][0]
+            tm_latest = latest_doc[MTIME]
+            tm_current = doc[MTIME]
             if format_date(tm_latest) == format_date(tm_current): # this change took place on the same day
-                item_groups[current_item_name].append(rev)
+                item_groups[current_item_name].append(doc)
         else:
-            item_groups[current_item_name] = [rev]
+            item_groups[current_item_name] = [doc]
 
     # Got the item dict, now doing grouping inside them
     editor_info = namedtuple('editor_info', ['editor', 'editor_revnos'])
-    for item_name, revs in item_groups.items():
+    for item_name, docs in item_groups.items():
         item_info = {}
         editors_info = OrderedDict()
         editors = []
         revnos = []
         comments = []
-        current_rev = revs[0]
+        current_doc = docs[0]
         item_info["item_name"] = item_name
-        item_info["timestamp"] = current_rev.timestamp
-        item_info["contenttype"] = current_rev.get(CONTENTTYPE)
-        item_info["action"] = current_rev.get(ACTION)
-        item_info["name"] = current_rev.get(NAME)
+        item_info["name"] = current_doc[NAME]
+        item_info["timestamp"] = current_doc[MTIME]
+        item_info["contenttype"] = current_doc[CONTENTTYPE]
+        item_info["action"] = current_doc[ACTION]
 
         # Aggregating comments, authors and revno
-        for rev in revs:
-            revnos.append(rev.revno)
-            comment = rev.get(COMMENT)
+        for doc in docs:
+            rev_no = doc["rev_no"]
+            revnos.append(rev_no)
+            comment = doc.get(COMMENT)
             if comment:
                 comment = "#%(revno)d %(comment)s" % {
-                          'revno': rev.revno,
+                          'revno': rev_no,
                           'comment': comment
                           }
                 comments.append(comment)
-            editor = get_editor_info(rev)
+            editor = get_editor_info(doc)
             editor_name = editor["name"]
             if editor_name in editors_info:
-                editors_info[editor_name].editor_revnos.append(rev.revno)
+                editors_info[editor_name].editor_revnos.append(rev_no)
             else:
-                editors_info[editor_name] = editor_info(editor, [rev.revno])
+                editors_info[editor_name] = editor_info(editor, [rev_no])
 
         if len(revnos) == 1:
             # there is only one change for this item in the history considered
@@ -797,7 +807,7 @@
     rev_tuple = namedtuple('rev_tuple', ['rev_date', 'item_revs'])
     rev_tuples = rev_tuple(prev_date, [])
     for item_group in item_groups.values():
-        tm = datetime.utcfromtimestamp(item_group["timestamp"])
+        tm = item_group["timestamp"]
         rev_date = format_date(tm)
         if revcount < offset:
             revcount += len(item_group["revnos"])
@@ -853,90 +863,52 @@
                            previous_offset=previous_offset,
                           )
 
-@frontend.route('/+wanteds')
-def wanted_items():
-    """ Returns a page with the list of non-existing items, which are wanted items and the
-        items they are linked or transcluded to helps show what items still need
-        to be written and shows whether there are any broken links. """
-    wanteds = _wanteds(flaskg.storage.iteritems())
-    item_name = request.values.get('item_name', '') # actions menu puts it into qs
-    return render_template('wanteds.html',
-                           headline=_(u'Wanted Items'),
-                           item_name=item_name,
-                           wanteds=wanteds)
+def _compute_item_sets():
+    """
+    compute sets of existing, linked, transcluded and no-revision item names
+    """
+    linked = set()
+    transcluded = set()
+    existing = set()
+    docs = flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)
+    for doc in docs:
+        existing.add(doc[NAME])
+        linked.update(doc.get(ITEMLINKS, []))
+        transcluded.update(doc.get(ITEMTRANSCLUSIONS, []))
+    return existing, linked, transcluded
 
 
-def _wanteds(items):
-    """
-    Returns a dict with all the names of non-existing items which are refed by
-    other items and the items which are refed by
-
-    :param items: all the items
-    :type items: iteratable sequence
-    :returns: a dict with all the wanted items and the items which are beign refed by
+@frontend.route('/+wanteds')
+def wanted_items():
     """
-    all_items = set()
-    wanteds = {}
-    for item in items:
-        current_item = item.name
-        all_items.add(current_item)
-        try:
-            current_rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            continue
-        # converting to sets so we can get the union
-        outgoing_links = current_rev.get(ITEMLINKS, [])
-        outgoing_transclusions = current_rev.get(ITEMTRANSCLUSIONS, [])
-        outgoing_refs = set(outgoing_transclusions + outgoing_links)
-        for refed_item in outgoing_refs:
-            if refed_item not in all_items:
-                if refed_item not in wanteds:
-                    wanteds[refed_item] = []
-                wanteds[refed_item].append(current_item)
-        if current_item in wanteds:
-            # if a previously wanted item has been found in the items storage, remove it
-            del wanteds[current_item]
-
-    return wanteds
+    Returns a list view of non-existing items that are linked to or
+    transcluded by other items. If you want to know by which items they are
+    referred to, use the backrefs functionality of the item in question.
+    """
+    existing, linked, transcluded = _compute_item_sets()
+    referred = linked | transcluded
+    wanteds = referred - existing
+    item_name = request.values.get('item_name', '') # actions menu puts it into qs
+    return render_template('item_link_list.html',
+                           headline=_(u'Wanted Items'),
+                           item_name=item_name,
+                           item_names=wanteds)
 
 
 @frontend.route('/+orphans')
 def orphaned_items():
-    """ Return a page with the list of items not being linked or transcluded
-        by any other items, that makes
-        them sometimes not discoverable. """
-    orphan = _orphans(flaskg.storage.iteritems())
+    """
+    Return a list view of existing items not being linked or transcluded
+    by any other item (which makes them sometimes not discoverable).
+    """
+    existing, linked, transcluded = _compute_item_sets()
+    referred = linked | transcluded
+    orphans = existing - referred
     item_name = request.values.get('item_name', '') # actions menu puts it into qs
     return render_template('item_link_list.html',
                            item_name=item_name,
                            headline=_(u'Orphaned Items'),
-                           item_names=orphan)
-
-
-def _orphans(items):
-    """
-    Returns a list with the names of all existing items not being refed by any other item
-
-    :param items: the list of all items
-    :type items: iteratable sequence
-    :returns: the list of all orphaned items
-    """
-    linked_items = set()
-    transcluded_items = set()
-    all_items = set()
-    norev_items = set()
-    for item in items:
-        all_items.add(item.name)
-        try:
-            current_rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            norev_items.add(item.name)
-        else:
-            linked_items.update(current_rev.get(ITEMLINKS, []))
-            transcluded_items.update(current_rev.get(ITEMTRANSCLUSIONS, []))
-    orphans = all_items - linked_items - transcluded_items - norev_items
-    logging.info("_orphans: Ignored %d item(s) that have no revisions" % len(norev_items))
-    return list(orphans)
+                           item_names=orphans)
 
 
 @frontend.route('/+quicklink/<itemname:item_name>')
@@ -1649,7 +1621,7 @@
     :rtype: tuple
     :returns: start word, end word, matches dict
     """
-    item_names = [item.name for item in flaskg.storage.iteritems()]
+    item_names = [doc[NAME] for doc in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)]
     if item_name in item_names:
         item_names.remove(item_name)
     # Get matches using wiki way, start and end of word
@@ -1814,13 +1786,18 @@
     """
     show a list or tag cloud of all tags in this wiki
     """
-    counts_tags_names = flaskg.storage.all_tags()
     item_name = request.values.get('item_name', '') # actions menu puts it into qs
-    if counts_tags_names:
-        # sort by tag name
-        counts_tags_names = sorted(counts_tags_names, key=lambda e: e[1])
+    docs = flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)
+    tags_counts = {}
+    for doc in docs:
+        tags = doc.get(TAGS, [])
+        logging.debug("name %s rev %s tags %s" % (doc[NAME], doc["rev_no"], tags))
+        for tag in tags:
+            tags_counts[tag] = tags_counts.setdefault(tag, 0) + 1
+    tags_counts = sorted(tags_counts.items())
+    if tags_counts:
         # this is a simple linear scaling
-        counts = [e[0] for e in counts_tags_names]
+        counts = [count for tags, count in tags_counts]
         count_min = min(counts)
         count_max = max(counts)
         weight_max = 9.99
@@ -1828,11 +1805,11 @@
             scale = weight_max / 2
         else:
             scale = weight_max / (count_max - count_min)
-        def cls(count, tag):
+        def cls(count):
             # return the css class for this tag
             weight = scale * (count - count_min)
             return "weight%d" % int(weight)  # weight0, ..., weight9
-        tags = [(cls(count, tag), tag) for count, tag, names in counts_tags_names]
+        tags = [(cls(count), tag) for tag, count in tags_counts]
     else:
         tags = []
     return render_template("global_tags.html",
@@ -1846,10 +1823,11 @@
     """
     show all items' names that have tag <tag>
     """
-    item_names = flaskg.storage.tagged_items(tag)
+    query = And([Term("wikiname", app.cfg.interwikiname), Term(TAGS, tag), ])
+    docs = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
+    item_names = [doc[NAME] for doc in docs]
     return render_template("item_link_list.html",
                            headline=_("Items tagged with %(tag)s", tag=tag),
                            item_name=tag,
                            item_names=item_names)
 
-
--- a/MoinMoin/apps/misc/views.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/apps/misc/views.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,17 +8,15 @@
 """
 
 
-import time
-
 from flask import Response
 from flask import current_app as app
 from flask import g as flaskg
 
 from MoinMoin.apps.misc import misc
 
+from MoinMoin.config import NAME, MTIME
 from MoinMoin.themes import render_template
 from MoinMoin import wikiutil
-from MoinMoin.storage.error import NoSuchRevisionError, NoSuchItemError
 
 SITEMAP_HAS_SYSTEM_ITEMS = True
 
@@ -27,18 +25,14 @@
     """
     Google (and others) XML sitemap
     """
-    def format_timestamp(ts):
-        return time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(ts))
+    def format_timestamp(dt):
+        return dt.strftime("%Y-%m-%dT%H:%M:%S+00:00")
 
     sitemap = []
-    for item in flaskg.storage.iteritems():
-        try:
-            rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            # XXX we currently also get user items, they have no revisions -
-            # but in the end, they should not be readable by the user anyways
-            continue
-        if wikiutil.isSystemItem(item.name):
+    for doc in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname):
+        name = doc[NAME]
+        mtime = doc[MTIME]
+        if False: # was: wikiutil.isSystemItem(name)   XXX add back later, when we have that in the index
             if not SITEMAP_HAS_SYSTEM_ITEMS:
                 continue
             # system items are rather boring
@@ -48,14 +42,13 @@
             # these are the content items:
             changefreq = "daily"
             priority = "0.5"
-        sitemap.append((item.name, format_timestamp(rev.timestamp), changefreq, priority))
+        sitemap.append((name, format_timestamp(mtime), changefreq, priority))
     # add an entry for root url
-    try:
-        item = flaskg.storage.get_item(app.cfg.item_root)
-        rev = item.get_revision(-1)
-        sitemap.append((u'', format_timestamp(rev.timestamp), "hourly", "1.0"))
-    except NoSuchItemError:
-        pass
+    root_item = app.cfg.item_root
+    docs = list(flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname, name=root_item))
+    if docs:
+        mtime = docs[0][MTIME]
+        sitemap.append((u'', format_timestamp(mtime), "hourly", "1.0"))
     sitemap.sort()
     content = render_template('misc/sitemap.xml', sitemap=sitemap)
     return Response(content, mimetype='text/xml')
@@ -70,8 +63,8 @@
     can implement SisterWiki functionality easily.
     See: http://usemod.com/cgi-bin/mb.pl?SisterSitesImplementationGuide
     """
-    # XXX we currently also get user items, fix this
-    item_names = sorted([item.name for item in flaskg.storage.iteritems()])
+    # XXX we currently also get trash items, fix this
+    item_names = sorted([doc[NAME] for doc in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)])
     content = render_template('misc/urls_names.txt', item_names=item_names)
     return Response(content, mimetype='text/plain')
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/auth/_tests/test_auth.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,72 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for auth.__init__
+"""
+
+from flask import g as flaskg
+
+from MoinMoin.auth import GivenAuth, handle_login, get_multistage_continuation_url
+from MoinMoin.user import create_user
+import pytest
+
+class TestGivenAuth(object):
+    """ Test: GivenAuth """
+    def test_decode_username(self):
+        givenauth_obj = GivenAuth()
+        result1 = givenauth_obj.decode_username('test_name')
+        assert result1 == u'test_name'
+        result2 = givenauth_obj.decode_username(123.45)
+        assert result2 == 123.45
+
+    def test_transform_username(self):
+        givenauth_obj = GivenAuth()
+        givenauth_obj.strip_maildomain = True
+        givenauth_obj.strip_windomain = True
+        givenauth_obj.titlecase = True
+        givenauth_obj.remove_blanks = True
+        result = givenauth_obj.transform_username(u'testDomain\\test name@moinmoin.org')
+        assert result == 'TestName'
+
+    def test_request(self):
+        givenauth_obj = GivenAuth()
+        flaskg.user.auth_method = 'given'
+        givenauth_obj.user_name = u'testDomain\\test_user@moinmoin.org'
+        givenauth_obj.strip_maildomain = True
+        givenauth_obj.strip_windomain = True
+        givenauth_obj.titlecase = True
+        givenauth_obj.remove_blanks = True
+        create_user('Test_User', 'test_pass', 'test@moinmoin.org')
+        test_user, bool_value = givenauth_obj.request(flaskg.user)
+        assert test_user.valid
+        assert test_user.name == u'Test_User'
+
+def test_handle_login():
+    # no messages in the biginning
+    assert not flaskg._login_messages
+    test_user1 = handle_login(flaskg.user, login_username = 'test_user', login_password = 'test_password', stage = 'moin')
+    test_login_message = [u'Invalid username or password.']
+    assert flaskg._login_messages == test_login_message
+    assert test_user1.name == u'anonymous'
+    assert not test_user1.valid
+    # pop the message
+    flaskg._login_messages.pop()
+    # try with a valid user
+    givenauth_obj = GivenAuth()
+    flaskg.user.auth_method = 'given'
+    givenauth_obj.user_name = u'Test_User'
+    create_user('Test_User', 'test_pass', 'test@moinmoin.org')
+    test_user, bool_value = givenauth_obj.request(flaskg.user)
+    test_user2 = handle_login(test_user, login_username = 'Test_User', login_password = 'test_pass', stage = 'moin')
+    assert not flaskg._login_messages
+    assert test_user2.name == u'Test_User'
+    assert test_user2.valid
+
+def test_get_multistage_continuation_url():
+    test_url = get_multistage_continuation_url('test_auth_name', extra_fields={'password': 'test_pass', 'test_key': 'test_value'})
+    assert 'test_key=test_value' in test_url
+    assert 'password=test_pass' in test_url
+    assert 'stage=test_auth_name' in test_url
+    assert 'login_submit=1' in test_url
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/auth/_tests/test_http.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,44 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for auth.http
+"""
+
+from flask import g as flaskg
+from flask import request
+
+from MoinMoin.user import create_user
+from MoinMoin.auth.http import HTTPAuthMoin
+import pytest
+
+class TestHTTPAuthMoin(object):
+    """ Test: HTTPAuthMoin """
+    class Auth:
+        def __init__(self):
+            self.username = 'ValidUser'
+            self.password = 'test_pass'
+
+    def setup_method(self, metod):
+        flaskg.user.auth_method = 'http'
+        request.authorization = self.Auth()
+
+    def teardown_method(self, method):
+        flaskg.user.auth_method = 'invalid'
+        request.authorization = None
+
+    def test_request(self):
+        # create a new user
+        create_user(u'ValidUser', 'test_pass', 'test_email@moinmoin')
+        httpauthmoin_obj = HTTPAuthMoin()
+        test_user, bool_val = httpauthmoin_obj.request(flaskg.user)
+        assert test_user.valid
+        assert test_user.name == u'ValidUser'
+        assert bool_val
+
+        # when auth_method is not 'http'
+        flaskg.user.auth_method = 'invalid'
+        test_user, bool_val = httpauthmoin_obj.request(flaskg.user)
+        assert not test_user.valid
+        assert test_user.name == u'anonymous'
+
--- a/MoinMoin/auth/_tests/test_ldap_login.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/auth/_tests/test_ldap_login.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,7 +7,7 @@
 """
 
 
-import py.test
+import pytest
 
 
 from MoinMoin._tests.ldap_testbase import LDAPTstBase, LdapEnvironment, check_environ, SLAPD_EXECUTABLE
@@ -18,7 +18,7 @@
 # first check if we have python 2.4, python-ldap and slapd:
 msg = check_environ()
 if msg:
-    py.test.skip(msg)
+    pytest.skip(msg)
 del msg
 
 import ldap
@@ -147,7 +147,7 @@
             ldap_env.create_env(slapd_config=self.slapd_config)
             started = ldap_env.start_slapd()
             if not started:
-                py.test.skip("Failed to start %s process, please see your syslog / log files"
+                pytest.skip("Failed to start %s process, please see your syslog / log files"
                              " (and check if stopping apparmor helps, in case you use it)." % SLAPD_EXECUTABLE)
             ldap_env.load_directory(ldif_content=self.ldif_content)
             self.ldap_envs.append(ldap_env)
@@ -188,7 +188,7 @@
             ldap_env.create_env(slapd_config=self.slapd_config)
             started = ldap_env.start_slapd()
             if not started:
-                py.test.skip("Failed to start %s process, please see your syslog / log files"
+                pytest.skip("Failed to start %s process, please see your syslog / log files"
                              " (and check if stopping apparmor helps, in case you use it)." % SLAPD_EXECUTABLE)
             ldap_env.load_directory(ldif_content=self.ldif_content)
             self.ldap_envs.append(ldap_env)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/auth/_tests/test_log.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,39 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for auth.log
+"""
+
+from MoinMoin import log
+logging = log.getLogger(__name__)
+
+from MoinMoin.auth.log import AuthLog
+from flask import g as flaskg
+import pytest
+
+class TestAuthLog(object):
+    """ Test: TestAuthLog """
+    def test_login(self):
+        authlog_obj = AuthLog()
+        result = authlog_obj.login(flaskg.user)
+        assert result.continue_flag
+        test_user_obj = result.user_obj
+        assert test_user_obj.name == u'anonymous'
+
+    def test_request(self):
+        authlog_obj = AuthLog()
+        result = authlog_obj.request(flaskg.user)
+        test_user, bool_value = result
+        assert test_user.name == u'anonymous'
+        assert not test_user.valid
+        assert bool_value
+
+    def test_logout(self):
+        authlog_obj = AuthLog()
+        result = authlog_obj.logout(flaskg.user)
+        test_user, bool_value = result
+        assert test_user.name == u'anonymous'
+        assert not test_user.valid
+        assert bool_value
+
--- a/MoinMoin/config/_tests/test_defaultconfig.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/config/_tests/test_defaultconfig.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py
+import pytest
 
 from flask import current_app as app
 
@@ -29,7 +29,7 @@
     def testBuiltinPasswordChecker(self):
         pw_checker = app.cfg.password_checker
         if not pw_checker:
-            py.test.skip("password_checker is disabled in the configuration, not testing it")
+            pytest.skip("password_checker is disabled in the configuration, not testing it")
         else:
             for pw, result in self.tests_builtin:
                 pw_error = pw_checker(self.username, pw)
--- a/MoinMoin/config/default.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/config/default.py	Thu Sep 01 00:16:33 2011 +0200
@@ -410,10 +410,6 @@
     ('namespace_mapping', None,
     "This needs to point to a (correctly ordered!) list of tuples, each tuple containing: Namespace identifier, backend, acl protection to be applied to that backend. " + \
     "E.g.: [('/', FSBackend('wiki/data'), dict(default='All:read,write,create')), ]. Please see HelpOnStorageConfiguration for further reference."),
-    ('load_xml', None,
-     'If this points to an xml file, the file is loaded into the storage backend(s) upon first request.'),
-    ('save_xml', None,
-     'If this points to an xml file, the current storage backend(s) content is saved into that file upon the first request.'),
   )),
   # ==========================================================================
   'items': ('Special item names', None, (
--- a/MoinMoin/conftest.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/conftest.py	Thu Sep 01 00:16:33 2011 +0200
@@ -23,46 +23,36 @@
                   '../wiki', # no tests there
                   '../instance', # tw likes to use this for wiki data (non-revisioned)
                  ]
-
 import atexit
 import os
 import sys
+import inspect
 
+import pytest
 import py
+import MoinMoin.log
+
+""" Logging for tests to avoid useless output like timing information on stderr on test failures
+"""
+Moindir = py.path.local(__file__).dirname
+config_file = Moindir + '/test_logging.conf'
+MoinMoin.log.load_config(config_file)
 
 from MoinMoin.app import create_app_ext, destroy_app, before_wiki, teardown_wiki
 from MoinMoin._tests import maketestwiki, wikiconfig
 from MoinMoin.storage.backends import create_simple_mapping
-
-coverage_modules = set()
-
-try:
-    """
-    This code adds support for coverage.py (see
-    http://nedbatchelder.com/code/modules/coverage.html).
-    It prints a coverage report for the modules specified in all
-    module globals (of the test modules) named "coverage_modules".
-    """
-
-    import coverage
+from flask import g as flaskg
 
-    def report_coverage():
-        coverage.stop()
-        module_list = sorted([sys.modules[mod] for mod in coverage_modules])
-        coverage.report(module_list)
+# In the beginning following variables have no values
+prev_app = None
+prev_cls = None
+prev_ctx = None
 
-    def callback(option, opt_str, value, parser):
-        atexit.register(report_coverage)
-        coverage.erase()
-        coverage.start()
-
-    py.test.config.addoptions('MoinMoin options', py.test.config.Option('-C',
-        '--coverage', action='callback', callback=callback,
-        help='Output information about code coverage (slow!)'))
-
-except ImportError:
-    coverage = None
-
+def get_previous(self_app, self_ctx, cls):
+    prev_app = self_app
+    prev_ctx = self_ctx
+    prev_cls = cls
+    return prev_app, prev_ctx, prev_cls
 
 def init_test_app(given_config):
     namespace_mapping = create_simple_mapping("memory:", given_config.content_acl)
@@ -82,61 +72,87 @@
     ctx.pop()
     destroy_app(app)
 
-
-class MoinClassCollector(py.test.collect.Class):
-
+class MoinTestFunction(pytest.collect.Function):
     def setup(self):
-        cls = self.obj
-        if hasattr(cls, 'Config'):
-            given_config = cls.Config
-        else:
-            given_config = wikiconfig.Config
-
-        def setup_method(f):
-            def wrapper(self, *args, **kwargs):
-                # Important: FIRST init the test app, then call the wrapped function.
-                self.app, self.ctx = init_test_app(given_config)
-                # Don't forget to call the class' setup_method if it has one.
-                return f(self, *args, **kwargs)
-            return wrapper
+        if inspect.isclass(self.parent.obj.__class__):
+            cls = self.parent.obj.__class__
 
-        def teardown_method(f):
-            def wrapper(self, *args, **kwargs):
-                # Don't forget to call the class' teardown_method if it has one.
-                # Important: FIRST call the wrapped function, so it can still
-                # access the stuff removed by deinit_test_app:
-                ret = f(self, *args, **kwargs)
-                deinit_test_app(self.app, self.ctx)
-                return ret
-            return wrapper
+            # global variables so that previous values can be accessed
+            global prev_app, prev_ctx, prev_cls
 
-        try:
-            # Wrap the actual setup_method in our decorator.
-            cls.setup_method = setup_method(cls.setup_method)
-        except AttributeError:
-            # Perhaps the test class did not define a setup_method.
-            def no_setup(self, method):
+            if hasattr(cls, 'Config'):
+                if prev_app is not None:
+                    # deinit previous app if previous app value is not None.
+                    deinit_test_app(prev_app, prev_ctx)
+                given_config = cls.Config
+                # init app
                 self.app, self.ctx = init_test_app(given_config)
-            cls.setup_method = no_setup
+            else:
+                given_config = wikiconfig.Config
+                # deinit the previous app if previous class had its own configuration
+                if hasattr(prev_cls, 'Config'):
+                    deinit_test_app(prev_app, prev_ctx)
 
-        try:
-            # Wrap the actual teardown_method in our decorator.
-            cls.teardown_method = teardown_method(cls.teardown_method)
-        except AttributeError:
-            # Perhaps the test class did not define a teardown_method.
-            def no_teardown(self, method):
-                deinit_test_app(self.app, self.ctx)
-            cls.teardown_method = no_teardown
+                # Initialize the app in following two conditions:
+                # 1. It is the first test item
+                # 2. Class of previous function item had its own configuration i.e. hasattr(cls, Config)
+                if prev_app is None or hasattr(prev_cls, 'Config'):
+                    self.app, self.ctx = init_test_app(given_config)
+                # continue assigning the values of the previous app and ctx to the current ones.
+                else:
+                    self.app = prev_app
+                    self.ctx = prev_ctx
 
-        super(MoinClassCollector, self).setup()
+            # Get the values from the function
+            prev_app, prev_ctx, prev_cls = get_previous(self.app, self.ctx, cls)
+
+        else:
+            prev_app, prev_ctx, prev_cls = get_previous(None, None, None)
+
+        super(MoinTestFunction, self).setup()
+        #XXX: hack till we get better funcarg tools
+        if hasattr(self._obj, 'im_self'):
+            self._obj.im_self.app = self.app
+
 
     def teardown(self):
-        super(MoinClassCollector, self).teardown()
+        clean_backend()
+        super(MoinTestFunction, self).teardown()
+
+    # Need to modify and add more stuffs
 
 
-class Module(py.test.collect.Module):
-    Class = MoinClassCollector
+def pytest_pycollect_makemodule(path, parent):
+    return Module(path, parent=parent)
 
+def pytest_pycollect_makeitem(__multicall__, collector, name, obj):
+    if collector.funcnamefilter(name) and inspect.isfunction(obj):
+        return MoinTestFunction(name, parent = collector)
+
+def pytest_pyfunc_call(pyfuncitem):
+    """hook to intercept generators and run them as a single test items"""
+    if inspect.isgeneratorfunction(pyfuncitem.obj):
+        for item in pyfuncitem.obj():
+            kwarg = item[1:]
+            item[0](*kwarg)
+
+def pytest_report_header(config):
+    return "The tests here are implemented only for pytest-2"
+
+def clean_backend():
+    """ method to cleanup the items created in testing process """
+    for test_item in flaskg.unprotected_storage.iteritems():
+        # some items don't have 'uuid' as key in them
+        # such items raise keyerror on test_item.destroy()
+        # add the key 'uuid' to such items
+        key_list = test_item.keys()
+        if not 'uuid' in key_list:
+            test_item.change_metadata()
+            test_item['uuid'] = 'temp_uuid'
+            test_item.publish_metadata()
+        test_item.destroy()
+
+class Module(pytest.collect.Module):
     def run(self, *args, **kwargs):
         if coverage is not None:
             coverage_modules.update(getattr(self.obj, 'coverage_modules', []))
--- a/MoinMoin/converter/__init__.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/__init__.py	Thu Sep 01 00:16:33 2011 +0200
@@ -77,57 +77,6 @@
         return self._register(self.Entry(factory, type_input, type_output, priority))
 
 
-from ..util.mime import Type, type_moin_document
-
-from MoinMoin.config import CONTENTTYPE
-
-from MoinMoin import log
-logging = log.getLogger(__name__)
-
-
-def convert_to_indexable(rev):
-    """
-    convert a revision to an indexable document
-
-    :param rev: item revision - please make sure that the content file is
-                ready to read all indexable content from it. if you have just
-                written that content or already read from it, you need to call
-                rev.seek(0) before calling convert_to_indexable(rev).
-    """
-    try:
-        # TODO use different converter mode?
-        # Maybe we want some special mode for the input converters so they emit
-        # different output than for normal rendering), esp. for the non-markup
-        # content types (images, etc.).
-        input_contenttype = rev[CONTENTTYPE]
-        output_contenttype = 'text/plain'
-        type_input_contenttype = Type(input_contenttype)
-        type_output_contenttype = Type(output_contenttype)
-        reg = default_registry
-        # first try a direct conversion (this could be useful for extraction
-        # of (meta)data from binary types, like from images or audio):
-        conv = reg.get(type_input_contenttype, type_output_contenttype)
-        if conv:
-            doc = conv(rev, input_contenttype)
-            return doc
-        # otherwise try via DOM as intermediate format (this is useful if
-        # input type is markup, to get rid of the markup):
-        input_conv = reg.get(type_input_contenttype, type_moin_document)
-        output_conv = reg.get(type_moin_document, type_output_contenttype)
-        if input_conv and output_conv:
-            doc = input_conv(rev, input_contenttype)
-            # We do not convert smileys, includes, macros, links, because
-            # it does not improve search results or even makes results worse.
-            doc = output_conv(doc)
-            return doc
-        # no way
-        raise TypeError("No converter for %s --> %s" % (input_contenttype, output_contenttype))
-    except Exception as e: # catch all exceptions, we don't want to break an indexing run
-        logging.exception("Exception happened in conversion:")
-        doc = u'ERROR [%s]' % str(e)
-        return doc
-
-
 default_registry = RegistryConverter()
 load_package_modules(__name__, __path__)
 
--- a/MoinMoin/converter/_tests/test__args.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test__args.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 from MoinMoin.converter._args import *
 
@@ -44,8 +44,8 @@
     assert a['keyword'] is None
     assert a['both'] is None
 
-    py.test.raises(IndexError, a.__getitem__, 2)
-    py.test.raises(KeyError, a.__getitem__, 'none')
+    pytest.raises(IndexError, a.__getitem__, 2)
+    pytest.raises(KeyError, a.__getitem__, 'none')
 
 def test_Arguments___len__():
     positional = ['positional', 'both']
--- a/MoinMoin/converter/_tests/test__args_wiki.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test__args_wiki.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 from MoinMoin.converter._args_wiki import *
 
--- a/MoinMoin/converter/_tests/test__wiki_macro.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test__wiki_macro.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 import re
 
 from MoinMoin.converter._args import Arguments
--- a/MoinMoin/converter/_tests/test_creole_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_creole_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 import re
 
--- a/MoinMoin/converter/_tests/test_docbook_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_docbook_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -10,11 +10,11 @@
 import re
 import StringIO
 
-import py.test
+import pytest
 try:
     from lxml import etree
 except:
-    py.test.skip("lxml module required to run test for docbook_in converter.")
+    pytest.skip("lxml module required to run test for docbook_in converter.")
 
 from emeraldtree.tree import *
 
--- a/MoinMoin/converter/_tests/test_docbook_out.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_docbook_out.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,11 +9,11 @@
 import re
 import StringIO
 
-import py.test
+import pytest
 try:
     from lxml import etree
 except:
-    py.test.skip("lxml module required to run test for docbook_out converter.")
+    pytest.skip("lxml module required to run test for docbook_out converter.")
 
 from emeraldtree.tree import *
 
--- a/MoinMoin/converter/_tests/test_html_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_html_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,11 +9,11 @@
 import re
 import StringIO
 
-import py.test
+import pytest
 try:
     from lxml import etree
 except:
-    py.test.skip("lxml module required to run test for html_in converter.")
+    pytest.skip("lxml module required to run test for html_in converter.")
 
 from emeraldtree.tree import *
 
--- a/MoinMoin/converter/_tests/test_html_in_out.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_html_in_out.py	Thu Sep 01 00:16:33 2011 +0200
@@ -12,11 +12,11 @@
 import StringIO
 import re
 
-import py.test
+import pytest
 try:
     from lxml import etree
 except:
-    py.test.skip("lxml module required to run test for html_in_out converter.")
+    pytest.skip("lxml module required to run test for html_in_out converter.")
 
 from MoinMoin import log
 logging = log.getLogger(__name__)
--- a/MoinMoin/converter/_tests/test_html_out.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_html_out.py	Thu Sep 01 00:16:33 2011 +0200
@@ -10,11 +10,11 @@
 import re
 import StringIO
 
-import py.test
+import pytest
 try:
     from lxml import etree
 except:
-    py.test.skip("lxml module required to run test for html_out converter.")
+    pytest.skip("lxml module required to run test for html_out converter.")
 
 from emeraldtree.tree import *
 
@@ -247,7 +247,7 @@
         for i in data:
             yield (self.do, ) + i
 
-    @py.test.mark.xfail
+    @pytest.mark.xfail
     def test_unknown(self):
         page = ET.XML("<page:unknown %s/>" % self.input_namespaces)
-        py.test.raises(ElementException, self.conv.__call__, page)
+        pytest.raises(ElementException, self.conv.__call__, page)
--- a/MoinMoin/converter/_tests/test_include.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_include.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 from MoinMoin.converter.include import *
 
@@ -48,3 +48,9 @@
     assert e.name == 'c'
     assert e.data == 'd'
 
+    x = XPointer('a(a(b))')
+    assert len(x) == 1
+    e = x[0]
+    assert e.name == 'a'
+    assert e.data == 'a(b)'
+
--- a/MoinMoin/converter/_tests/test_link.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_link.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 from emeraldtree import tree as ET
 
@@ -67,14 +67,14 @@
 
     def _do_wiki(self, input, output, skip=None):
         if skip:
-            py.test.skip(skip)
+            pytest.skip(skip)
         elem = ET.Element(None)
         self.conv.handle_wiki_links(elem, Iri(input))
         assert elem.get(xlink.href) == output
 
     def _do_wikilocal(self, input, page, output, skip=None):
         if skip:
-            py.test.skip(skip)
+            pytest.skip(skip)
         elem = ET.Element(None)
         self.conv.handle_wikilocal_links(elem, Iri(input), Iri(page))
         assert elem.get(xlink.href) == output
--- a/MoinMoin/converter/_tests/test_mediawiki_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_mediawiki_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,7 +7,7 @@
 """
 
 
-import py.test
+import pytest
 import re
 
 from MoinMoin.converter.mediawiki_in import *
--- a/MoinMoin/converter/_tests/test_moinwiki19_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_moinwiki19_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,11 +6,11 @@
 """
 
 
-import py.test
+import pytest
 
-from ..moinwiki19_in import ConverterFormat19
+from MoinMoin.converter.moinwiki19_in import ConverterFormat19
 
-from .test_moinwiki_in import TestConverter as _Base
+from test_moinwiki_in import TestConverter as _Base
 
 
 class TestConverterFormat19(_Base):
--- a/MoinMoin/converter/_tests/test_moinwiki_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_moinwiki_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,13 +6,13 @@
 """
 
 
-import py.test
+import pytest
 
 import re
 
 from MoinMoin.util.tree import moin_page, xlink
 
-from ..moinwiki_in import Converter
+from MoinMoin.converter.moinwiki_in import Converter
 
 
 class TestConverter(object):
@@ -312,6 +312,6 @@
 
     def do(self, input, output, args={}, skip=None):
         if skip:
-            py.test.skip(skip)
+            pytest.skip(skip)
         out = self.conv(input, 'text/x.moin.wiki;charset=utf-8', **args)
         assert self.serialize(out) == output
--- a/MoinMoin/converter/_tests/test_moinwiki_in_out.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_moinwiki_in_out.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,7 +9,7 @@
 """
 
 
-import py.test
+import pytest
 import re
 
 from emeraldtree import ElementTree as ET
@@ -519,7 +519,7 @@
 
     def do(self, input, output, args={}, skip=None):
         if skip:
-            py.test.skip(skip)
+            pytest.skip(skip)
         out = self.conv_in(input, 'text/x.moin.wiki;charset=utf-8', **args)
         out = self.conv_out(self.handle_input(self.serialize(out)), **args)
         assert self.handle_output(out) == output
--- a/MoinMoin/converter/_tests/test_moinwiki_out.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_moinwiki_out.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,7 +7,7 @@
 """
 
 
-import py.test
+import pytest
 import re
 
 from MoinMoin.converter.moinwiki_out import *
--- a/MoinMoin/converter/_tests/test_rst_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_rst_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,7 +7,7 @@
 """
 
 
-import py.test
+import pytest
 import re
 
 from MoinMoin.converter.rst_in import *
--- a/MoinMoin/converter/_tests/test_rst_out.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_rst_out.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 import re
 
 from MoinMoin.converter.rst_out import *
--- a/MoinMoin/converter/_tests/test_smiley.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/_tests/test_smiley.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,7 +7,7 @@
 
 
 import re
-import py
+import pytest
 
 from MoinMoin.converter.smiley import Converter, moin_page, ET
 
@@ -66,7 +66,7 @@
 
 
 def test_smiley_convert(input, query):
-    etree = py.test.importorskip('lxml.etree')
+    etree = pytest.importorskip('lxml.etree')
     conv = Converter()
     print 'input:', input
     out_elem = conv(ET.XML(input))
--- a/MoinMoin/converter/include.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/include.py	Thu Sep 01 00:16:33 2011 +0200
@@ -1,5 +1,5 @@
 # Copyright: 2008 MoinMoin:BastianBlank
-# Copyright: 2010 MoinMoin:ThomasWaldmann
+# Copyright: 2010-2011 MoinMoin:ThomasWaldmann
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
@@ -17,8 +17,12 @@
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
+from flask import current_app as app
 from flask import g as flaskg
 
+from whoosh.query import Term, And, Wildcard
+
+from MoinMoin.config import NAME
 from MoinMoin import wikiutil
 from MoinMoin.items import Item
 from MoinMoin.util.mime import type_moin_document
@@ -154,7 +158,7 @@
 
                     if xp_include:
                         for entry in xp_include:
-                            name, data = entry.name, entry.data
+                            name, data = entry.name, entry.data_unescape
                             if name == 'pages':
                                 xp_include_pages = data
                             elif name == 'sort':
@@ -197,13 +201,16 @@
                     pages = ((page, link), )
 
                 elif xp_include_pages:
-                    # We have a regex of pages to include
-                    from MoinMoin.storage.terms import NameFn
-                    inc_match = re.compile(xp_include_pages)
-                    root_item = Item(name=u'')
-                    pagelist = sorted([item.name for item in root_item.list_items(NameFn(inc_match))])
-                    if xp_include_sort == 'descending':
-                        pagelist.reverse()
+                    # XXX we currently interpret xp_include_pages as wildcard, but it should be regex
+                    # for compatibility with moin 1.9. whoosh has upcoming regex support, but it is not
+                    # released yet.
+                    if xp_include_pages.startswith('^'):
+                        # get rid of the leading ^ the Include macro needed to get into "regex mode"
+                        xp_include_pages = xp_include_pages[1:]
+                    query = And([Term("wikiname", app.cfg.interwikiname), Wildcard("name_exact", xp_include_pages)])
+                    reverse = xp_include_sort == 'descending'
+                    results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", reverse=reverse, limit=None)
+                    pagelist = [result[NAME] for result in results]
                     if xp_include_skipitems is not None:
                         pagelist = pagelist[xp_include_skipitems:]
                     if xp_include_items is not None:
--- a/MoinMoin/converter/rst_in.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/converter/rst_in.py	Thu Sep 01 00:16:33 2011 +0200
@@ -18,6 +18,7 @@
 from __future__ import absolute_import, division
 
 import re
+import pytest
 
 from MoinMoin import log
 logging = log.getLogger(__name__)
@@ -31,6 +32,7 @@
 from ._util import decode_data, normalize_split_text
 
 #### TODO: try block (do not crash if we don't have docutils)
+pytest.importorskip('docutils')
 from docutils import nodes, utils, writers, core
 from docutils.parsers.rst import Parser
 from docutils.nodes import reference, literal_block
--- a/MoinMoin/datastruct/backends/_tests/__init__.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/datastruct/backends/_tests/__init__.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,7 +9,7 @@
 """
 
 
-from py.test import raises
+from pytest import raises
 
 from flask import current_app as app
 from flask import g as flaskg
--- a/MoinMoin/datastruct/backends/_tests/test_composite_dicts.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/datastruct/backends/_tests/test_composite_dicts.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,8 +7,6 @@
 """
 
 
-from py.test import raises
-
 from MoinMoin.datastruct.backends._tests import DictsBackendTest
 from MoinMoin.datastruct import ConfigDicts, CompositeDicts, DictDoesNotExistError
 from MoinMoin._tests import wikiconfig
--- a/MoinMoin/datastruct/backends/_tests/test_composite_groups.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/datastruct/backends/_tests/test_composite_groups.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-from py.test import raises
+from pytest import raises
 
 from flask import g as flaskg
 
--- a/MoinMoin/datastruct/backends/_tests/test_config_dicts.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/datastruct/backends/_tests/test_config_dicts.py	Thu Sep 01 00:16:33 2011 +0200
@@ -19,6 +19,12 @@
             dicts = DictsBackendTest.dicts
             return ConfigDicts(dicts)
 
+    def test__iter__(self):
+        ConfigDicts_obj = ConfigDicts(DictsBackendTest.dicts)
+        test_keyiterator = ConfigDicts.__iter__(ConfigDicts_obj)
+        expected = ['SomeTestDict', 'SomeOtherTestDict']
+        for result in test_keyiterator:
+            assert result in expected
 
 coverage_modules = ['MoinMoin.datastruct.backends.config_dicts']
 
--- a/MoinMoin/datastruct/backends/_tests/test_wiki_dicts.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/datastruct/backends/_tests/test_wiki_dicts.py	Thu Sep 01 00:16:33 2011 +0200
@@ -13,7 +13,6 @@
 from MoinMoin.datastruct.backends import wiki_dicts
 from MoinMoin.config import SOMEDICT
 from MoinMoin._tests import become_trusted, update_item
-
 DATA = "This is a dict item."
 
 
@@ -35,6 +34,11 @@
                     u"Two": u"2"}
         update_item(u'SomeOtherTestDict', 0, {SOMEDICT: somedict}, DATA)
 
+    def test__retrieve_items(self):
+        wikidict_obj = wiki_dicts.WikiDicts()
+        result = wiki_dicts.WikiDicts._retrieve_items(wikidict_obj, u'SomeOtherTestDict')
+        expected = {u'Two': u'2', u'One': u'1'}
+        assert result == expected
 
 coverage_modules = ['MoinMoin.datastruct.backends.wiki_dicts']
 
--- a/MoinMoin/datastruct/backends/_tests/test_wiki_groups.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/datastruct/backends/_tests/test_wiki_groups.py	Thu Sep 01 00:16:33 2011 +0200
@@ -10,7 +10,7 @@
 """
 
 
-import py
+import pytest
 
 from flask import current_app as app
 from flask import g as flaskg
@@ -46,13 +46,13 @@
         result = u'ExampleUser' in flaskg.groups[u'AnotherGroup']
         assert result
 
-        py.test.raises(GroupDoesNotExistError, lambda: flaskg.groups[u'SomeGroup'])
+        pytest.raises(GroupDoesNotExistError, lambda: flaskg.groups[u'SomeGroup'])
 
     def test_copy_group_item(self):
         """
         Tests copying a group item.
         """
-        py.test.skip("item.copy() is not finished")
+        pytest.skip("item.copy() is not finished")
 
         become_trusted()
         item = update_item(u'SomeGroup', 0,  {USERGROUP: ["ExampleUser"]}, DATA)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/i18n/_tests/test_i18n.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,34 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for i18n
+"""
+
+from MoinMoin.i18n import get_locale, get_timezone
+import pytest
+
+from MoinMoin.i18n import _, L_, N_
+
+def test_user_attributes():
+    test_locale = get_locale()
+    assert test_locale == 'en'
+
+    test_timezone = get_timezone()
+    assert test_timezone == 'UTC'
+
+def test_text():
+    # test for gettext
+    result = _('test_text')
+    assert result == 'test_text'
+
+    # test for lazy_gettext
+    result = L_('test_lazy_text')
+    assert result == u'test_lazy_text'
+
+    # test for ngettext
+    result1 = N_('text1', 'text2', 1)
+    assert result1 == 'text1'
+    result2 = N_('text1', 'text2', 2)
+    assert result2 == 'text2'
+
--- a/MoinMoin/items/__init__.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/items/__init__.py	Thu Sep 01 00:16:33 2011 +0200
@@ -25,6 +25,9 @@
 
 from flatland import Form, String, Integer, Boolean, Enum
 from flatland.validation import Validator, Present, IsEmail, ValueBetween, URLValidator, Converted
+
+from whoosh.query import Term, And, Prefix
+
 from MoinMoin.util.forms import FileStorage
 
 from MoinMoin.security.textcha import TextCha, TextChaizedForm, TextChaValid
@@ -502,75 +505,26 @@
             newrev[CONTENTTYPE] = unicode(contenttype_current or contenttype_guessed or 'application/octet-stream')
 
         newrev[ACTION] = unicode(action)
-        self.before_revision_commit(newrev, data)
         storage_item.commit()
         item_modified.send(app._get_current_object(), item_name=name)
         return new_rev_no, size
 
-    def before_revision_commit(self, newrev, data):
-        """
-        hook that can be used to add more meta data to a revision before
-        it is committed.
-
-        :param newrev: new (still uncommitted) revision - modify as wanted
-        :param data: either str or open file (we can avoid having to read/seek
-                     rev's data with this)
-        """
-        remote_addr = request.remote_addr
-        if remote_addr:
-            if app.cfg.log_remote_addr:
-                newrev[ADDRESS] = unicode(remote_addr)
-                hostname = wikiutil.get_hostname(remote_addr)
-                if hostname:
-                    newrev[HOSTNAME] = hostname
-        if flaskg.user.valid:
-            newrev[USERID] = unicode(flaskg.user.id)
-
-    def search_items(self, term=None):
-        """ search items matching the term or,
-            if term is None, return all items
-        """
-        if term:
-            backend_items = flaskg.storage.search_items(term)
-        else:
-            # special case: we just want all items
-            backend_items = flaskg.storage.iteritems()
-        for item in backend_items:
-            yield Item.create(item=item)
-
-    list_items = search_items  # just for cosmetics
-
-    def count_items(self, term=None):
-        """
-        Return item count for matching items. See search_items() for details.
-        """
-        count = 0
-        # we intentionally use a loop to avoid creating a list with all item objects:
-        for item in self.list_items(term):
-            count += 1
-        return count
-
     def get_index(self):
         """ create an index of sub items of this item """
-        import re
-        from MoinMoin.storage.terms import NameRE
-
         if self.name:
             prefix = self.name + u'/'
+            query = And([Term("wikiname", app.cfg.interwikiname), Prefix("name_exact", prefix)])
         else:
             # trick: an item of empty name can be considered as "virtual root item",
             # that has all wiki items as sub items
             prefix = u''
-        sub_item_re = u"^%s.*" % re.escape(prefix)
-        regex = re.compile(sub_item_re, re.UNICODE)
-
-        item_iterator = self.search_items(NameRE(regex))
-
+            query = Term("wikiname", app.cfg.interwikiname)
+        results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
         # We only want the sub-item part of the item names, not the whole item objects.
         prefix_len = len(prefix)
-        items = [(item.name, item.name[prefix_len:], item.meta.get(CONTENTTYPE))
-                 for item in item_iterator]
-        return sorted(items)
+        items = [(result[NAME], result[NAME][prefix_len:], result[CONTENTTYPE])
+                 for result in results]
+        return items
 
     def flat_index(self, startswith=None, selected_groups=None):
         """
@@ -705,13 +659,12 @@
 
     def get_templates(self, contenttype=None):
         """ create a list of templates (for some specific contenttype) """
-        from MoinMoin.storage.terms import AND, LastRevisionMetaDataMatch
-        term = LastRevisionMetaDataMatch(TAGS, ['template']) # XXX there might be other tags
-        if contenttype:
-            term = AND(term, LastRevisionMetaDataMatch(CONTENTTYPE, contenttype))
-        item_iterator = self.search_items(term)
-        items = [item.name for item in item_iterator]
-        return sorted(items)
+        terms = [Term("wikiname", app.cfg.interwikiname), Term(TAGS, u'template')]
+        if contenttype is not None:
+            terms.append(Term(CONTENTTYPE, contenttype))
+        query = And(terms)
+        results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
+        return [result[NAME] for result in results]
 
     def do_modify(self, contenttype, template_name):
         # XXX think about and add item template support
@@ -1220,33 +1173,6 @@
     some kind of item with markup
     (internal links and transcluded items)
     """
-    def before_revision_commit(self, newrev, data):
-        """
-        add ITEMLINKS and ITEMTRANSCLUSIONS metadata
-        """
-        super(MarkupItem, self).before_revision_commit(newrev, data)
-
-        if hasattr(data, "read"):
-            data.seek(0)
-            data = data.read()
-        elif isinstance(data, str):
-            pass
-        else:
-            raise StorageError("unsupported content object: %r" % data)
-
-        from MoinMoin.converter import default_registry as reg
-
-        input_conv = reg.get(Type(self.contenttype), type_moin_document)
-        item_conv = reg.get(type_moin_document, type_moin_document, items='refs')
-
-        i = Iri(scheme='wiki', authority='', path='/' + self.name)
-
-        doc = input_conv(self.rev, self.contenttype)
-        doc.set(moin_page.page_href, unicode(i))
-        doc = item_conv(doc)
-
-        newrev[ITEMLINKS] = item_conv.get_links()
-        newrev[ITEMTRANSCLUSIONS] = item_conv.get_transclusions()
 
 
 class MoinWiki(MarkupItem):
--- a/MoinMoin/items/_tests/test_Item.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/items/_tests/test_Item.py	Thu Sep 01 00:16:33 2011 +0200
@@ -5,21 +5,23 @@
     MoinMoin - MoinMoin.items Tests
 """
 
+# TODO: spilt the tests into multiple ones after the item.__init__ is split.
 
-import py
+import pytest
 
 from flask import g as flaskg
 
 from MoinMoin._tests import become_trusted
-from MoinMoin.items import Item, ApplicationXTar, NonExistent, Binary, Text, Image, TransformableBitmapImage
+from MoinMoin.items import Item, ApplicationXTar, NonExistent, Binary, Text, Image, TransformableBitmapImage, MarkupItem
 from MoinMoin.config import CONTENTTYPE, ADDRESS, COMMENT, HOSTNAME, USERID, ACTION
 
 class TestItem(object):
+
     def testNonExistent(self):
-        item = Item.create('DoesNotExist')
+        item = Item.create(u'DoesNotExist')
         assert isinstance(item, NonExistent)
         meta, data = item.meta, item.data
-        assert meta == {CONTENTTYPE: 'application/x-nonexistent'}
+        assert meta == {CONTENTTYPE: u'application/x-nonexistent'}
         assert data == ''
 
     def testClassFinder(self):
@@ -30,7 +32,7 @@
                 (u'image/tiff', Image),
                 (u'image/png', TransformableBitmapImage),
             ]:
-            item = Item.create('foo', contenttype=contenttype)
+            item = Item.create(u'foo', contenttype=contenttype)
             assert isinstance(item, ExpectedClass)
 
     def testCRUD(self):
@@ -117,10 +119,140 @@
                                   (u'Foo/mn', u'mn', 'image/jpeg', False),
                                   ]
 
+
+    def test_meta_filter(self):
+        name = u'Test_item'
+        contenttype = u'text/plain;charset=utf-8'
+        meta = {'test_key': 'test_val', CONTENTTYPE: contenttype, 'name': 'test_name', 'uuid': 'test_uuid'}
+        item = Item.create(name)
+        result = Item.meta_filter(item, meta)
+        # keys like NAME and UUID are filtered
+        expected = {'test_key': 'test_val', CONTENTTYPE: contenttype}
+        assert result == expected
+
+    def test_meta_dict_to_text(self):
+        name = u'Test_item'
+        contenttype = u'text/plain;charset=utf-8'
+        meta = {'test_key': 'test_val', CONTENTTYPE: contenttype, 'name': 'test_name', 'uuid': 'test_uuid'}
+        item = Item.create(name)
+        result = Item.meta_dict_to_text(item, meta)
+        expected = '{\n  "contenttype": "text/plain;charset=utf-8", \n  "test_key": "test_val"\n}'
+        assert result == expected
+
+    def test_meta_text_to_dict(self):
+        name = u'Test_item'
+        contenttype = u'text/plain;charset=utf-8'
+        text = '{\n  "contenttype": "text/plain;charset=utf-8", \n  "test_key": "test_val", \n "name": "test_name", \n "uuid": "test_uuid"\n}'
+        item = Item.create(name)
+        result = Item.meta_text_to_dict(item, text)
+        expected = {'test_key': 'test_val', CONTENTTYPE: contenttype}
+        assert result == expected
+
+    def test_rename(self):
+        name = u'Test_Item'
+        contenttype = u'text/plain;charset=utf-8'
+        data = 'test_data'
+        meta = {'test_key': 'test_value', CONTENTTYPE: contenttype}
+        comment = u'saved it'
+        become_trusted()
+        item = Item.create(name)
+        item._save(meta, data, comment=comment)
+        item = Item.create(name)
+        # item and its contents before renaming
+        assert item.name == u'Test_Item'
+        assert item.meta['comment'] == u'saved it'
+        Item.rename(item, u'Test_new_Item', comment=u'renamed')
+        new_name = u'Test_new_Item'
+        item = Item.create(new_name)
+        # item and its contents after renaming
+        assert item.name == u'Test_new_Item'
+        assert item.meta['comment'] == u'renamed'
+        assert item.meta['name_old'] == u'Test_Item'
+        assert item.data == u'test_data'
+
+    def test_delete(self):
+        name = u'Test_Item'
+        contenttype = u'text/plain;charset=utf-8'
+        data = 'test_data'
+        meta = {'test_key': 'test_value', CONTENTTYPE: contenttype}
+        comment = u'saved it'
+        item = Item.create(name)
+        item._save(meta, data, comment=comment)
+        item = Item.create(name)
+        item.delete(u'item deleted')
+        # item and its contents after deletion
+        item = Item.create(name)
+        assert item.name == u'Test_Item'
+        assert item.meta == {'contenttype': 'application/x-nonexistent'}
+
+    def test_revert(self):
+        name = u'Test_Item'
+        contenttype = u'text/plain;charset=utf-8'
+        data = 'test_data'
+        meta = {'test_key': 'test_value', CONTENTTYPE: contenttype}
+        comment = u'saved it'
+        item = Item.create(name)
+        item._save(meta, data, comment=comment)
+        item = Item.create(name)
+        item.revert()
+        item = Item.create(name)
+        assert item.meta['action'] == u'REVERT'
+
+    def test_modify(self):
+        name = u'Test_Item'
+        contenttype = u'text/plain;charset=utf-8'
+        data = 'test_data'
+        meta = {'test_key': 'test_value', CONTENTTYPE: contenttype}
+        comment = u'saved it'
+        item = Item.create(name)
+        item._save(meta, data, comment=comment)
+        item = Item.create(name)
+        assert item.name == u'Test_Item'
+        assert item.meta['test_key'] == 'test_value'
+        # call item.modify
+        item.modify()
+        item = Item.create(name)
+        assert item.name == u'Test_Item'
+        with pytest.raises(KeyError):
+            item.meta['test_key']
+
+
+class TestBinary(object):
+    """ Test for arbitrary binary items """
+
+    def test_get_templates(self):
+        item_name1 = u'Template_Item1'
+        item1 = Binary.create(item_name1)
+        contenttype1 = u'text/plain'
+        meta = {CONTENTTYPE: contenttype1, 'tags': ['template']}
+        item1._save(meta)
+        item1 = Binary.create(item_name1)
+
+        item_name2 = u'Template_Item2'
+        item2 = Binary.create(item_name2)
+        contenttype1 = u'text/plain'
+        meta = {CONTENTTYPE: contenttype1, 'tags': ['template']}
+        item2._save(meta)
+        item2 = Binary.create(item_name2)
+
+        item_name3 = u'Template_Item3'
+        item3 = Binary.create(item_name3)
+        contenttype2 = u'image/png'
+        meta = {CONTENTTYPE: contenttype2, 'tags': ['template']}
+        item3._save(meta)
+        item3 = Binary.create(item_name3)
+        # two items of same content type
+        result1 = item1.get_templates(contenttype1)
+        assert result1 == [item_name1, item_name2]
+        # third of different content type
+        result2 = item1.get_templates(contenttype2)
+        assert result2 == [item_name3]
+
 class TestTarItems(object):
     """
     tests for the container items
     """
+
     def testCreateContainerRevision(self):
         """
         creates a container and tests the content saved to the container
@@ -158,5 +290,125 @@
         item = Item.create(item_name, contenttype=u'application/x-tar')
         assert item.get_member('example1.txt').read() == filecontent
 
+class TestZipMixin(object):
+    """ Test for zip-like items """
+
+    def test_put_member(self):
+        item_name = u'Zip_file'
+        item = Item.create(item_name, contenttype='application/zip')
+        filecontent = 'test_contents'
+        content_length = len(filecontent)
+        members = set(['example1.txt', 'example2.txt'])
+        with pytest.raises(NotImplementedError):
+            item.put_member('example1.txt', filecontent, content_length, expected_members=members)
+
+class TestTransformableBitmapImage(object):
+
+    def test__transform(self):
+        item_name = u'image_Item'
+        item = Binary.create(item_name)
+        contenttype = u'image/jpeg'
+        meta = {CONTENTTYPE: contenttype}
+        item._save(meta)
+        item = Binary.create(item_name)
+        try:
+            from PIL import Image as PILImage
+            with pytest.raises(ValueError):
+                result = TransformableBitmapImage._transform(item, 'plane/text')
+        except ImportError:
+            result = TransformableBitmapImage._transform(item, contenttype)
+            assert result == (u'image/jpeg', '')
+
+    def test__render_data_diff(self):
+        item_name = u'image_Item'
+        item = Binary.create(item_name)
+        contenttype = u'image/jpeg'
+        meta = {CONTENTTYPE: contenttype}
+        item._save(meta)
+        item1 = Binary.create(item_name)
+        try:
+            from PIL import Image as PILImage
+            result = TransformableBitmapImage._render_data_diff(item1, item.rev, item1.rev)
+            expected = '<img src="/+diffraw/image_Item?rev2=0" />'
+            assert str(result) == expected
+        except ImportError:
+            # no PIL
+            pass
+
+    def test__render_data_diff_text(self):
+        item_name = u'image_Item'
+        item = Binary.create(item_name)
+        contenttype = u'image/jpeg'
+        meta = {CONTENTTYPE: contenttype}
+        item._save(meta)
+        item1 = Binary.create(item_name)
+        data = 'test_data'
+        comment = u'next revision'
+        item1._save(meta, data, comment=comment)
+        item2 = Binary.create(item_name)
+        try:
+            from PIL import Image as PILImage
+            result = TransformableBitmapImage._render_data_diff_text(item1, item1.rev, item2.rev)
+            expected = u'The items have different data.'
+            assert result == expected
+        except ImportError:
+            pass
+
+class TestText(object):
+
+    def test_data_conversion(self):
+        item_name = u'Text_Item'
+        item = Text.create(item_name, u'text/plane')
+        test_text = u'This \n is \n a \n Test'
+        # test for data_internal_to_form
+        result = Text.data_internal_to_form(item, test_text)
+        expected = u'This \r\n is \r\n a \r\n Test'
+        assert result == expected
+        # test for data_form_to_internal
+        test_form = u'This \r\n is \r\n a \r\n Test'
+        result = Text.data_form_to_internal(item, test_text)
+        expected = test_text
+        assert result == expected
+        # test for data_internal_to_storage
+        result = Text.data_internal_to_storage(item, test_text)
+        expected = 'This \r\n is \r\n a \r\n Test'
+        assert result == expected
+        # test for data_storage_to_internal
+        data_storage = 'This \r\n is \r\n a \r\n Test'
+        result = Text.data_storage_to_internal(item, data_storage)
+        expected = test_text
+        assert result == expected
+
+    def test__render_data_diff_text(self):
+        item_name = u'Text_Item'
+        item = Text.create(item_name)
+        contenttype = u'text/plane'
+        meta = {CONTENTTYPE: contenttype}
+        item._save(meta)
+        item1 = Text.create(item_name)
+        data = 'test_data'
+        comment = u'next revision'
+        item1._save(meta, data, comment=comment)
+        item2 = Text.create(item_name)
+        result = Text._render_data_diff_text(item1, item1.rev, item2.rev)
+        expected = u'- \n+ test_data'
+        assert result == expected
+        assert item2.data == ''
+
+    def test__render_data_highlight(self):
+        item_name = u'Text_Item'
+        item = Text.create(item_name)
+        contenttype = u'text/plane'
+        meta = {CONTENTTYPE: contenttype}
+        item._save(meta)
+        item1 = Text.create(item_name)
+        data = 'test_data\nnext line'
+        comment = u'next revision'
+        item1._save(meta, data, comment=comment)
+        item2 = Text.create(item_name)
+        result = Text._render_data_highlight(item2)
+        assert u'<pre class="highlight">test_data\n' in result
+        assert item2.data == ''
+
 coverage_modules = ['MoinMoin.items']
 
--- a/MoinMoin/macro/_base.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/macro/_base.py	Thu Sep 01 00:16:33 2011 +0200
@@ -58,7 +58,7 @@
     to nothing.
     """
     def __call__(self, content, arguments, page_url, alternative, context_block):
-        if not content_block:
+        if not context_block:
             return self.macro(content, arguments, page_url, alternative)
 
 class MacroPageLinkListBase(MacroBlockBase):
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/macro/_tests/test_Anchor.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,21 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for macro.Anchor
+"""
+
+import pytest
+from MoinMoin.macro.Anchor import *
+
+def test_Macro():
+    macro_obj = Macro()
+    with pytest.raises(ValueError):
+        macro_obj.macro('content', None, 'page_url', 'alternative')
+
+    arguments = [('test_argument1', 'test_argument2'), 'test_argumnt3']
+    result = macro_obj.macro('content', arguments, 'page_url', 'alternative')
+    test_anchor = result.attrib.values()
+    # test_anchor[0] since it returns a list
+    assert test_anchor[0] == arguments[0]
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/macro/_tests/test_Date.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,43 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for macro.Date
+"""
+
+import time
+from datetime import datetime
+
+from flaskext.babel import format_date, format_datetime
+
+from MoinMoin.macro.Date import MacroDateTimeBase, Macro
+import pytest
+
+class TestMacroDateTimeBase(object):
+    def test_parse_time(self):
+        MacroDateTimeBase_obj = MacroDateTimeBase()
+        test_time_args = '2011-08-07T11:11:11+0533'
+        result = MacroDateTimeBase_obj.parse_time(test_time_args)
+        expected = 1312695491.0
+        assert result == expected
+        result = format_datetime(datetime.utcfromtimestamp(result))
+        expected = u'Aug 7, 2011 5:38:11 AM'
+        assert result == expected
+        with pytest.raises(ValueError):
+            # things after next 10,000 years can't be predicted
+            MacroDateTimeBase_obj.parse_time('12011-08-07T11:11:11')
+
+class TestMacro(object):
+    def test_macro(self):
+        macro_obj = Macro()
+        # when arguments is None
+        result = macro_obj.macro('content', None, 'page_url', 'alternative')
+        test_time = time.time()
+        test_time = format_date(datetime.utcfromtimestamp(test_time))
+        assert result == test_time
+
+        arguments = ['2011-08-07T11:11:11+0533', 'argument2']
+        result = macro_obj.macro('content', arguments, 'page_url', 'alternative')
+        expected = u'Aug 7, 2011'
+        assert result == expected
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/macro/_tests/test_DateTime.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,33 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for macro.DateTime
+"""
+
+import time
+from datetime import datetime
+
+from MoinMoin.macro.DateTime import *
+import pytest
+
+def test_Macro():
+    """Test: DateTime.Macro """
+
+    macro_obj = Macro()
+    # when arguments is None
+    result = macro_obj.macro('content', None, 'page_url', 'alternative')
+    # get the current time
+    test_time = time.time()
+    test_time = format_datetime(datetime.utcfromtimestamp(test_time))
+    assert test_time == result
+
+    arguments = ['2011-08-07T11:11:11', 'argument2']
+    result  = macro_obj.macro('content', arguments, 'page_url', 'alternative')
+    expected = u'Aug 7, 2011 11:11:11 AM'
+    assert result == expected
+
+    arguments = ['incorrect_argument']
+    with pytest.raises(ValueError):
+        macro_obj.macro('content', arguments, 'page_url', 'alternative')
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/macro/_tests/test_GetText.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,19 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for macro.GetText
+"""
+
+import pytest
+from MoinMoin.converter._args import Arguments
+from MoinMoin.macro.GetText import *
+
+def test_Macro():
+    """ test for Macro.macro """
+    macro_obj = Macro()
+    arguments = Arguments(['test_argument1', 'test_argument2'])
+    result = macro_obj.macro('content', arguments, 'page_url', 'alternative')
+    expected = u'test_argument1 test_argument2'
+    assert result == expected
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/macro/_tests/test_GetVal.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,52 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for macro.GetVal
+"""
+
+from flask import g as flaskg
+
+from MoinMoin.macro.GetVal import *
+from MoinMoin.config import SOMEDICT
+from MoinMoin._tests import become_trusted, update_item
+from MoinMoin.conftest import init_test_app, deinit_test_app
+from MoinMoin._tests import wikiconfig
+import pytest
+DATA = "This is a dict item."
+
+class TestMacro(object):
+    """ Test: GetVal.Macro """
+
+    def setup_method(self, method):
+        # temporary hack till we apply test cleanup mechanism on tests.
+        self.app, self.ctx = init_test_app(wikiconfig.Config)
+        become_trusted()
+        somedict = {u"One": u"1",
+                    u"Two": u"2"}
+        update_item(u'TestDict', 0, {SOMEDICT: somedict}, DATA)
+
+    def teardown_method(self, method):
+        deinit_test_app(self.app, self.ctx)
+
+    def test_Macro(self):
+        macro_obj = Macro()
+        arguments = [u'TestDict']
+        with pytest.raises(ValueError):
+            macro_obj.macro('content', arguments, 'page_url', 'alternative')
+
+        # add the second element to arguments
+        arguments.append(u'One')
+
+        if not flaskg.user.may.read(arguments[0]):
+            with pytest.raises(ValueError):
+                macro_obj.macro('content', arguments, 'page_url', 'alternative')
+
+        result = macro_obj.macro('content', arguments, 'page_url', 'alternative')
+        assert result == u'1'
+
+        # change the value of second element
+        arguments[1] = u'Two'
+        result = macro_obj.macro('content', arguments, 'page_url', 'alternative')
+        assert result == u'2'
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/macro/_tests/test_Verbatim.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,14 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for macro.Verbatim
+"""
+
+from MoinMoin.macro.Verbatim import *
+
+def test_Macro():
+    macro_obj = Macro()
+    result = macro_obj.macro(u'test_text')
+    assert result == u'test_text'
+
--- a/MoinMoin/macro/_tests/test__base.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/macro/_tests/test__base.py	Thu Sep 01 00:16:33 2011 +0200
@@ -1,61 +1,60 @@
-# Copyright: 2008 MoinMoin:BastianBlank
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
 MoinMoin - Tests for MoinMoin.macro._base
 """
 
-
-import py.test
-py.test.skip("test is out of sync with tested code")
-
+import pytest
 from MoinMoin.macro._base import *
 
-def test_MacroBase___init__():
-    request = object()
-
-    m = MacroBase(request, None, 'alt', 'context')
-
-    assert m.immutable is False
-    assert m.alt == 'alt'
-    assert m.context == 'context'
-
-def test_MacroBlockBase___call__():
-    item = u'text'
-
-    class Test(MacroBlockBase):
-        def call_macro(self, content):
-            return item
-
-    r = Test(None, None, 'alt', 'block')()
-    assert r is item
-
-    r = Test(None, None, 'alt', 'inline')()
-    assert r == 'alt'
+class TestMacroBase(object):
+    """ Test for Macro base and related classes """
 
-def test_MacroInlineBase___call__():
-    item = u'text'
-
-    class Test(MacroInlineBase):
-        def call_macro(self, content):
-            return item
-
-    r = Test(None, None, 'alt', 'block')()
-    assert r[0] is item
-
-    r = Test(None, None, 'alt', 'inline')()
-    assert r is item
+    def test_MacroBase(self):
+        """ test for MacroBase class """
+        macrobase_obj  = MacroBase()
+        assert not macrobase_obj.immutable
+        with pytest.raises(NotImplementedError):
+            macrobase_obj.__call__('content', 'arguments', 'page_url', 'alternative', 'context_block')
 
-def test_MacroInlineOnlyBase___call__():
-    item = u'text'
 
-    class Test(MacroInlineOnlyBase):
-        def call_macro(self, content):
-            return item
+    def test_MacroBlockBase(self):
+        """ test for MacroBlockBase class """
+        class Test_MacroBlockBase(MacroBlockBase):
+            """ inherited class from MacroBlockBase """
+            def __init__(self):
+                self.alt = 'alt returned'
 
-    r = Test(None, None, 'alt', 'block')()
-    assert r is None
+        macroblockbase_obj = Test_MacroBlockBase()
+        result = macroblockbase_obj.__call__('content', 'arguments', 'page_url', 'alternative', context_block = False)
+        assert result == 'alt returned'
+        with pytest.raises(NotImplementedError):
+            result = macroblockbase_obj.__call__('content', 'arguments', 'page_url', 'alternative', 'context_block')
 
-    r = Test(None, None, 'alt', 'inline')()
-    assert r is item
+    def test_MacroInlineBase(self):
+        """ test for MacroInlineBase class """
+        class Test_MacroInlineBase(MacroInlineBase):
+            """ inherited class from MacroInlineBase """
+            def macro(self, content, arguments, page_url, alternative):
+                return 'test_macro'
 
+        macroinlinebase_obj = Test_MacroInlineBase()
+        result = macroinlinebase_obj.__call__('content', 'arguments', 'page_url', 'alternative', context_block = False)
+        assert result == 'test_macro'
+        result = macroinlinebase_obj.__call__('content', 'arguments', 'page_url', 'alternative', 'context_block')
+        assert result.text == 'test_macro'
+        result.remove('test_macro')
+        assert not result.text
+
+    def test_MacroInlineOnlyBase(self):
+        """ test for MacroInlineOnlyBase class """
+        class Test_MacroInlineOnlyBase(MacroInlineOnlyBase):
+            """ inherited class from MacroInlineOnlyBase """
+            def macro(self, content, arguments, page_url, alternative):
+                return 'test_macro'
+
+        macroinlineonlybase_obj = Test_MacroInlineOnlyBase()
+        result = macroinlineonlybase_obj.__call__('content', 'arguments', 'page_url', 'alternative', context_block = False)
+        assert result == 'test_macro'
+
--- a/MoinMoin/script/maint/index.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/script/maint/index.py	Thu Sep 01 00:16:33 2011 +0200
@@ -20,7 +20,7 @@
 from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError
 from MoinMoin.util.mime import Type
 from MoinMoin.search.indexing import backend_to_index
-from MoinMoin.converter import convert_to_indexable
+from MoinMoin.storage.backends.indexing import convert_to_indexable
 
 from MoinMoin import log
 logging = log.getLogger(__name__)
@@ -60,8 +60,16 @@
             Building in app.cfg.index_dir_tmp
             """
             indexnames = [indexname for indexname, schema in indexnames_schemas]
-            with MultiSegmentWriter(all_rev_index, procs, limitmb) as all_rev_writer:
-                with MultiSegmentWriter(latest_rev_index, procs, limitmb) as latest_rev_writer:
+            if procs == 1:
+                # MultiSegmentWriter sometimes has issues and is pointless for procs == 1,
+                # so use the simple writer when --procs 1 is given:
+                _all_rev_writer = all_rev_index.writer()
+                _latest_rev_writer = latest_rev_index.writer()
+            else:
+                _all_rev_writer = MultiSegmentWriter(all_rev_index, procs, limitmb)
+                _latest_rev_writer = MultiSegmentWriter(latest_rev_index, procs, limitmb)
+            with _all_rev_writer as all_rev_writer:
+                with _latest_rev_writer as latest_rev_writer:
                     for item in backend.iter_items_noindex():
                         try:
                             rev_no = None
@@ -78,7 +86,7 @@
                         except NoSuchRevisionError: # item has no such revision
                             continue
                         # revision is now the latest revision of this item
-                        if "latest_revisions_index" in indexnames and rev_no:
+                        if "latest_revisions_index" in indexnames and rev_no is not None:
                             metadata = backend_to_index(revision, rev_no, latest_rev_schema, rev_content, interwikiname)
                             latest_rev_writer.add_document(**metadata)
 
--- a/MoinMoin/script/maint/reduce_revisions.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/script/maint/reduce_revisions.py	Thu Sep 01 00:16:33 2011 +0200
@@ -1,37 +1,44 @@
 # Copyright: 2009 MoinMoin:ChristopherDenter
 # Copyright: 2011 MoinMoin:ReimarBauer
+# Copyright: 2011 MoinMoin:ThomasWaldmann
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-    MoinMoin - Reduce Revisions of a backend
+MoinMoin - Reduce Revisions of a backend
 
-    This script removes all revisions but the last one from all selected items.
+This script removes all revisions but the last one from all selected items.
 """
 
 
-import re
 from flask import current_app as app
 from flaskext.script import Command, Option
 
-from MoinMoin.storage.terms import NameRE
+from MoinMoin.config import NAME
 
 
 class Reduce_Revisions(Command):
     description = "This command can be used to remove all revisions but the last one from all selected items."
     option_list = (
-        Option('--pattern', '-p', required=False, dest='pattern', type=unicode, default=".*",
-               help="You can limit the operation on certain items whose names match the given pattern."),
+        Option('--query', '-q', dest="query", type=unicode, default='',
+               help='Only perform the operation on items found by the given query.'),
     )
 
-    def run(self, pattern):
+    def run(self, query):
         storage = app.unprotected_storage
-        query = NameRE(re.compile(pattern))
-        # If no pattern is given, the default regex will match every item.
-        for item in storage.search_items(query):
+        if query:
+            qp = storage.query_parser(["name_exact", ], all_revs=False)
+            q = qp.parse(query)
+        else:
+            q = Every()
+        results = storage.search(q, all_revs=False, limit=None)
+        for result in results:
+            item_name = result[NAME]
+            item = storage.get_item(item_name)
             current_revno = item.next_revno - 1
             for revno in item.list_revisions():
                 if revno < current_revno:
                     rev = item.get_revision(revno)
+                    print "Destroying %r revision %d." % (item_name, revno)
                     rev.destroy()
 
         print "Finished reducing backend."
--- a/MoinMoin/script/maint/set_meta.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/script/maint/set_meta.py	Thu Sep 01 00:16:33 2011 +0200
@@ -1,16 +1,16 @@
 # Copyright: 2009 MoinMoin:ChristopherDenter
 # Copyright: 2011 MoinMoin:ReimarBauer
+# Copyright: 2011 MoinMoin:ThomasWaldmann
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-    MoinMoin - Set Metadata of a revision
+MoinMoin - Set Metadata of a revision
 
-    This script duplicates the last revision of the selected item
-    and sets or removes metadata.
+This script duplicates the last revision of the selected item
+and sets or removes metadata.
 """
 
 
-import re
 from ast import literal_eval
 from shutil import copyfileobj
 
@@ -18,32 +18,40 @@
 from flask import g as flaskg
 from flaskext.script import Command, Option
 
+from MoinMoin.config import NAME
 from MoinMoin.script import fatal
-from MoinMoin.storage.terms import NameRE
 from MoinMoin.storage.error import NoSuchRevisionError
 
+
 class Set_Meta(Command):
     description = "This command can be used to set meta data of a new revision."
     option_list = (
         Option('--key', '-k', required=False, dest='key', type=unicode,
                help="The key you want to set/change in the new revision"),
-        Option('--value', '-v', dest="text", type=unicode,
+        Option('--value', '-v', dest="value", type=unicode,
                help='The value to set for the given key.'),
         Option('--remove', '-r', dest="remove", action='store_true', default=False,
                help='If you want to delete the key given, add this flag.'),
-        Option('--pattern', '-p', dest="pattern", type=unicode, default='.*',
-               help='Only perform the operation on items whose names match the pattern.')
+        Option('--query', '-q', dest="query", type=unicode, default='',
+               help='Only perform the operation on items found by the given query.')
     )
 
-    def run(self, key, text, remove, pattern):
+    def run(self, key, value, remove, query):
         storage = app.unprotected_storage
 
-        if not ((key and text) or (key and remove)) or (key and text and remove):
+        if not ((key and value) or (key and remove)) or (key and value and remove):
             fatal("You need to either specify a proper key/value pair or "
                   "only a key you want to delete (with -r set).")
 
-        query = NameRE(re.compile(pattern))
-        for item in storage.search_items(query):
+        if query:
+            qp = storage.query_parser(["name_exact", ], all_revs=False)
+            q = qp.parse(query)
+        else:
+            q = Every()
+        results = storage.search(q, all_revs=False, limit=None)
+        for result in results:
+            item_name = result[NAME]
+            item = storage.get_item(item_name)
             try:
                 last_rev = item.get_revision(-1)
             except NoSuchRevisionError:
@@ -62,7 +70,10 @@
 
             if not remove:
                 # Set or overwrite given metadata key with text
-                value = literal_eval(text)
-                next_rev[key] = value
+                next_rev[key] = literal_eval(value)
+                print "Processing %r, setting %s=%r." % (item_name, key, value)
+            else:
+                print "Processing %r, removing %s." % (item_name, key)
+
             item.commit()
 
--- a/MoinMoin/search/_tests/test_analyzers.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/search/_tests/test_analyzers.py	Thu Sep 01 00:16:33 2011 +0200
@@ -119,12 +119,20 @@
 
     test_cases_query = [
                   # (query, tokens)
-                  (u'text/plain', [u'text', u'plain']),
-                  (u'text/plain;charset=utf-8', [u'text', u'plain', u'charset=utf-8']),
+                  (u'text/plain',
+                   [u'text/plain', u'text', u'plain']),
+                  (u'text/plain;charset=utf-8',
+                   [u'text/plain;charset=utf-8', u'text', u'plain', u'charset=utf-8']),
                   (u'text/html;value1=foo;value2=bar',
-                   [u'text', u'html', u'value1=foo', u'value2=bar'],
+                   [u'text/html;value1=foo;value2=bar', u'text', u'html', u'value1=foo', u'value2=bar'],
                   ),
-                  (u'text/html;value1=foo;value1=bar', [u'text', u'html', u'value1=bar'])
+                  # we normalize, sort the params:
+                  (u'text/html;value2=bar;value1=foo',
+                   [u'text/html;value1=foo;value2=bar', u'text', u'html', u'value1=foo', u'value2=bar'],
+                  ),
+                  # later values for same key overwrite earlier ones:
+                  (u'text/html;value1=foo;value1=bar',
+                   [u'text/html;value1=bar', u'text', u'html', u'value1=bar'])
                  ]
 
     def make_tokenizer(self):
--- a/MoinMoin/search/analyzers.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/search/analyzers.py	Thu Sep 01 00:16:33 2011 +0200
@@ -20,10 +20,10 @@
         Tokenizer behaviour:
 
         Input: u"text/x.moin.wiki;charset=utf-8"
-        Output: u"text", u"x.moin.wiki", u"charset=utf-8"
+        Output: u"text/x.moin.wiki;charset=utf-8", u"text", u"x.moin.wiki", u"charset=utf-8"
 
         Input: u"application/pdf"
-        Output: u"application", u"pdf"
+        Output: u"application/pdf", u"application", u"pdf"
 
         :param value: String for tokenization
         :param start_pos: The position number of the first token. For example,
@@ -37,6 +37,16 @@
         pos = start_pos
         tk = Token()
         tp = Type(value)
+        # we need to yield the complete contenttype in one piece,
+        # so we can find it with Term(CONTENTTYPE, contenttype):
+        if tp.type is not None and tp.subtype is not None:
+            # note: we do not use "value" directly, so Type.__unicode__ can normalize it:
+            tk.text = unicode(tp)
+            if positions:
+                tk.pos = pos
+                pos += 1
+            yield tk
+        # now yield the pieces:
         tk.text = tp.type
         if positions:
             tk.pos = pos
--- a/MoinMoin/search/indexing.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/search/indexing.py	Thu Sep 01 00:16:33 2011 +0200
@@ -34,7 +34,7 @@
     doc = dict([(str(key), value)
                 for key, value in backend_rev.items()
                 if key in schema])
-    doc[MTIME] = datetime.datetime.fromtimestamp(backend_rev[MTIME])
+    doc[MTIME] = datetime.datetime.utcfromtimestamp(backend_rev[MTIME])
     doc["name_exact"] = backend_rev[NAME]
     doc["rev_no"] = rev_no
     doc["wikiname"] = wikiname
@@ -64,36 +64,68 @@
         self._index_dir = index_dir or self._cfg.index_dir
 
         common_fields = dict(
+            # wikiname so we can have a shared index in a wiki farm, always check this!
+            # taken from app.cfg.interwikiname
             wikiname=ID(stored=True),
+            # tokenized NAME from metadata - use this for manual searching from UI
             name=TEXT(stored=True, multitoken_query="and", analyzer=item_name_analyzer(), field_boost=2.0),
+            # unmodified NAME from metadata - use this for precise lookup by the code.
+            # also needed for wildcard search, so the original string as well as the query
+            # (with the wildcard) is not cut into pieces.
             name_exact=ID(field_boost=3.0),
+            # revision number, integer 0..n
             rev_no=NUMERIC(stored=True),
+            # MTIME from revision metadata (converted to UTC datetime)
             mtime=DATETIME(stored=True),
+            # tokenized CONTENTTYPE from metadata
             contenttype=TEXT(stored=True, multitoken_query="and", analyzer=MimeTokenizer()),
+            # unmodified list of TAGS from metadata
             tags=ID(stored=True),
+            # LANGUAGE from metadata
             language=ID(stored=True),
+            # USERID from metadata
             userid=ID(stored=True),
+            # ADDRESS from metadata
             address=ID(stored=True),
+            # HOSTNAME from metadata
             hostname=ID(stored=True),
+            # SIZE from metadata
+            size=NUMERIC(stored=True),
+            # ACTION from metadata
+            action=ID(stored=True),
+            # tokenized COMMENT from metadata
+            comment=TEXT(stored=True, multitoken_query="and"),
+            # data (content), converted to text/plain and tokenized
             content=TEXT(stored=True, multitoken_query="and"),
         )
+        latest_revs_fields = dict(
+            # UUID from metadata - as there is only latest rev of same item here, it is unique
+            uuid=ID(unique=True, stored=True),
+            # unmodified list of ITEMLINKS from metadata
+            itemlinks=ID(stored=True),
+            # unmodified list of ITEMTRANSCLUSIONS from metadata
+            itemtransclusions=ID(stored=True),
+            # tokenized ACL from metadata
+            acl=TEXT(analyzer=AclTokenizer(self._cfg), multitoken_query="and", stored=True),
+            **common_fields
+        )
 
-        self.latest_revisions_schema = Schema(uuid=ID(unique=True, stored=True),
-                                              itemlinks=ID(stored=True),
-                                              itemtransclusions=ID(stored=True),
-                                              acl=TEXT(analyzer=AclTokenizer(self._cfg), multitoken_query="and", stored=True),
-                                              **common_fields)
+        all_revs_fields = dict(
+            # UUID from metadata
+            uuid=ID(stored=True),
+            **common_fields
+        )
 
-        self.all_revisions_schema = Schema(uuid=ID(stored=True),
-                                           **common_fields)
+        self.latest_revisions_schema = Schema(**latest_revs_fields)
+        self.all_revisions_schema = Schema(**all_revs_fields)
 
         # Define dynamic fields
-        dynamic_fields = [("*_id", ID),
-                          ("*_text", TEXT),
-                          ("*_keyword", KEYWORD),
-                          ("*_numeric", NUMERIC),
-                          ("*_datetime", DATETIME),
-                          ("*_boolean", BOOLEAN)
+        dynamic_fields = [("*_id", ID(stored=True)),
+                          ("*_text", TEXT(stored=True)),
+                          ("*_keyword", KEYWORD(stored=True)),
+                          ("*_numeric", NUMERIC(stored=True)),
+                          ("*_datetime", DATETIME(stored=True)),
+                          ("*_boolean", BOOLEAN(stored=True)),
                          ]
 
         # Adding dynamic fields to schemas
--- a/MoinMoin/security/_tests/test_security.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/security/_tests/test_security.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,7 +8,7 @@
 """
 
 
-import py
+import pytest
 
 from flask import current_app as app
 
@@ -33,12 +33,12 @@
     def testEmpty(self):
         """ security: empty acl string raise StopIteration """
         acl_iter = acliter('')
-        py.test.raises(StopIteration, acl_iter.next)
+        pytest.raises(StopIteration, acl_iter.next)
 
     def testWhiteSpace(self):
         """ security: white space acl string raise StopIteration """
         acl_iter = acliter('       ')
-        py.test.raises(StopIteration, acl_iter.next)
+        pytest.raises(StopIteration, acl_iter.next)
 
     def testDefault(self):
         """ security: default meta acl """
@@ -137,7 +137,7 @@
         mod, entries, rights = acl_iter.next()
         assert entries == ['UserOne']
         assert rights == ['read']
-        py.test.raises(StopIteration, acl_iter.next)
+        pytest.raises(StopIteration, acl_iter.next)
 
     def testEmptyNamesWithRight(self):
         """ security: empty names with rights
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/security/_tests/test_textcha.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,115 @@
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+Test for security.textcha
+"""
+
+from flask import current_app as app
+from flask import g as flaskg
+
+from MoinMoin.security.textcha import TextCha, TextChaValid, TextChaizedForm
+import pytest
+
+class TestTextCha(object):
+    """ Test: class TextCha """
+    def setup_method(self, method):
+        cfg = app.cfg
+        cfg.textchas = {'test_user_locale':
+                            {'Good Question': 'Good Answer',
+                            'What is the question?': 'Test_Answer'}
+                       }
+        cfg.secrets['security/textcha'] = "test_secret"
+        flaskg.user.locale = 'test_user_locale'
+
+    def teardown_method(self, method):
+        cfg = app.cfg
+        cfg.textchas = None
+        cfg.secrets.pop('security/textcha')
+        flaskg.user.locale = None
+
+    def test_textcha(self):
+        """ test for textchas and its attributes """
+        test_form = TextChaizedForm()
+        test_form['textcha_question'].value = None
+
+        textcha_obj = TextCha(test_form)
+
+        # test for textcha
+        test_textchas = textcha_obj.textchas
+        expected_textchas  = {'Good Question': 'Good Answer',
+                                'What is the question?': 'Test_Answer'}
+        assert test_textchas == expected_textchas
+        # test for the question
+        test_question = textcha_obj.question
+        possible_questions = ['Good Question', 'What is the question?']
+        assert test_question in possible_questions
+        # test for answer_re
+        possible_answers = ['Good Answer', 'Test_Answer']
+        result_answer1 = textcha_obj.answer_re.match(expected_textchas[test_question])
+        test_answer = result_answer1.group()
+        assert test_answer in possible_answers
+        # invalid value
+        result_answer2 = textcha_obj.answer_re.match('Bad Answer')
+        assert not result_answer2
+        # test for answer_regex
+        result_answer = textcha_obj.answer_regex
+        assert result_answer in possible_answers
+
+        # when question is specified earlier
+        test_signature = 'fb5a8cc203b07b66637aafa7b0647da17e249e9c'
+        test_form['textcha_question'].value = 'What is the question? 9876543210' + test_signature
+        textcha_obj = TextCha(test_form)
+        # test for the question
+        test_question = textcha_obj.question
+        expected_question = 'What is the question?'
+        assert test_question == expected_question
+        # test the answer
+        test_answer = textcha_obj.answer_regex
+        assert test_answer == 'Test_Answer'
+        assert test_signature == textcha_obj.signature
+        assert textcha_obj.timestamp == 9876543210
+
+    def test_amend_form(self):
+        # textchas are disabled for 'some_locale'
+        flaskg.user.locale = 'some_locale'
+        test_form = TextChaizedForm()
+        test_form['textcha_question'].value = None
+        textcha_obj = TextCha(test_form)
+        # before calling amend_form
+        assert not textcha_obj.form['textcha_question'].optional
+        assert not textcha_obj.form['textcha'].optional
+        # on calling amend_form
+        textcha_obj.amend_form()
+        assert textcha_obj.form['textcha_question'].optional
+        assert textcha_obj.form['textcha'].optional
+
+class TestTextChaValid(object):
+    """ Test: class TextChaValid """
+    def setup_method(self, method):
+        cfg = app.cfg
+        cfg.textchas = {'test_user_locale':
+                            {'Good Question': 'Good Answer'}
+                       }
+        cfg.secrets['security/textcha'] = "test_secret"
+        flaskg.user.locale = 'test_user_locale'
+
+    def teardown_method(self, method):
+        cfg = app.cfg
+        cfg.textchas = None
+        cfg.secrets.pop('security/textcha')
+        flaskg.user.locale = None
+
+    class Element:
+        def __init__(self):
+            self.parent = None
+            self.value = 'Good Answer'
+
+    def test_validate(self):
+        test_form = TextChaizedForm()
+        textchavalid_obj = TextChaValid()
+        test_element = self.Element()
+        test_element.parent = test_form
+        result = textchavalid_obj.validate(test_element, 'test_state')
+        assert result
+
--- a/MoinMoin/storage/__init__.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/__init__.py	Thu Sep 01 00:16:33 2011 +0200
@@ -62,7 +62,7 @@
     This class abstracts access to backends. If you want to write a specific
     backend, say a mercurial backend, you have to implement the methods below.
     A backend knows of its items and can perform several item related operations
-    such as search_items, get_item, create_item, etc.
+    such as get_item, create_item, etc.
     """
     #
     # If you need to write a backend it is sufficient
@@ -87,22 +87,6 @@
         """
         pass
 
-    def search_items(self, searchterm):
-        """
-        Takes a MoinMoin search term and returns an iterator (maybe empty) over
-        matching item objects (NOT item names!).
-
-        :type searchterm: MoinMoin search term
-        :param searchterm: The term for which to search.
-        :rtype: iterator of item objects
-        """
-        # Very simple implementation because we have no indexing
-        # or anything like that. If you want to optimize this, override it.
-        for item in self.iteritems():
-            searchterm.prepare()
-            if searchterm.evaluate(item):
-                yield item
-
     def get_item(self, itemname):
         """
         Returns item object or raises Exception if that item does not exist.
@@ -287,8 +271,6 @@
         a wiki item, as such a deletion does not really delete anything from disk but
         just hides the former existence of the item. Such a deletion is undoable, while
         having destroyed an item is not.
-        This also destroys all history related to the item. In particular, this also
-        deletes all the item's revisions and they won't turn up in history any longer.
 
         In case the item has already been destroyed by someone else (e.g. another process)
         this method should just pass silently as the job is already done.
@@ -759,7 +741,6 @@
     that defaults to None for newly created revisions in which case it will be
     assigned at commit() time. It is writable for use by converter backends, but
     care must be taken in that case to create monotone timestamps!
-    This timestamp is also retrieved via the backend's history() method.
     """
     def __init__(self, item, revno):
         """
--- a/MoinMoin/storage/_tests/test_backends.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends.py	Thu Sep 01 00:16:33 2011 +0200
@@ -16,14 +16,13 @@
 """
 
 
-import py.test, re, time
+import pytest, re, time
 
 from flask import g as flaskg
 
 from MoinMoin.storage import Item, NewRevision
 from MoinMoin.storage.backends import memory
 from MoinMoin.storage.error import NoSuchItemError, ItemAlreadyExistsError, NoSuchRevisionError, RevisionAlreadyExistsError
-from MoinMoin.storage import terms
 from MoinMoin.config import SIZE
 
 item_names = (u"quite_normal",
@@ -109,11 +108,11 @@
     def test_item_rename_to_existing(self):
         item1 = self.create_rev_item_helper(u"fresh_item")
         item2 = self.create_rev_item_helper(u"try to rename")
-        py.test.raises(ItemAlreadyExistsError, item1.rename, item2.name)
+        pytest.raises(ItemAlreadyExistsError, item1.rename, item2.name)
 
     def rename_item_invalid_name(self, name, newname):
         item = self.backend.create_item(name)
-        py.test.raises(TypeError, item.rename, newname)
+        pytest.raises(TypeError, item.rename, newname)
 
     def test_item_rename_to_invalid(self):
         for num, invalid_name in enumerate(self.invalid_names):
@@ -129,7 +128,7 @@
         assert len(item1.list_revisions()) == 2
 
     def create_item_invalid_name(self, name):
-        py.test.raises(TypeError, self.backend.create_item, name)
+        pytest.raises(TypeError, self.backend.create_item, name)
 
     def test_create_item_wrong_itemname(self):
         for item_name in self.invalid_names:
@@ -153,14 +152,14 @@
 
     def test_create_rev_item_again(self):
         self.create_rev_item_helper(u"item1")
-        py.test.raises(ItemAlreadyExistsError, self.backend.create_item, u"item1")
+        pytest.raises(ItemAlreadyExistsError, self.backend.create_item, u"item1")
 
     def test_create_meta_item_again(self):
         self.create_meta_item_helper(u"item2")
-        py.test.raises(ItemAlreadyExistsError, self.backend.create_item, u"item2")
+        pytest.raises(ItemAlreadyExistsError, self.backend.create_item, u"item2")
 
     def test_get_item_that_doesnt_exist(self):
-        py.test.raises(NoSuchItemError, self.backend.get_item, u"i_do_not_exist")
+        pytest.raises(NoSuchItemError, self.backend.get_item, u"i_do_not_exist")
 
     def test_has_item(self):
         self.create_rev_item_helper(u"versioned")
@@ -171,33 +170,6 @@
     def test_has_item_that_doesnt_exist(self):
         assert not self.backend.has_item(u"i_do_not_exist")
 
-    def test_search_simple(self):
-        for name in [u"songlist", u"song lyric", u"odd_SONG_item"]:
-            self.create_rev_item_helper(name)
-        self.create_meta_item_helper(u"new_song_player")
-        query_string = u"song"
-        query = terms.Name(query_string, True)
-        for num, item in enumerate(self.backend.search_items(query)):
-            assert item.name.find(query_string) != -1
-        assert num == 2
-
-    def test_search_better(self):
-        self.create_rev_item_helper(u'abcde')
-        self.create_rev_item_helper(u'abcdef')
-        self.create_rev_item_helper(u'abcdefg')
-        self.create_rev_item_helper(u'abcdefgh')
-
-        def _test_search(term, expected):
-            found = list(self.backend.search_items(term))
-            assert len(found) == expected
-
-        # must be /part/ of the name
-        yield _test_search, terms.Name(u'AbCdEf', False), 3
-        yield _test_search, terms.Name(u'AbCdEf', True), 0
-        yield _test_search, terms.Name(u'abcdef', True), 3
-        yield _test_search, terms.NameRE(re.compile(u'abcde.*')), 4
-        yield _test_search, terms.NameFn(lambda n: n == u'abcdef'), 1
-
     def test_iteritems_1(self):
         for num in range(10, 20):
             self.create_rev_item_helper(u"item_" + str(num).zfill(2))
@@ -412,13 +384,13 @@
     def test_mixed_commit_metadata1(self):
         item = self.backend.create_item(u'mixed1')
         item.create_revision(0)
-        py.test.raises(RuntimeError, item.change_metadata)
+        pytest.raises(RuntimeError, item.change_metadata)
         item.rollback()
 
     def test_mixed_commit_metadata2(self):
         item = self.backend.create_item(u'mixed2')
         item.change_metadata()
-        py.test.raises(RuntimeError, item.create_revision, 0)
+        pytest.raises(RuntimeError, item.create_revision, 0)
 
     def test_item_metadata_change_and_publish(self):
         item = self.backend.create_item(u"test item metadata change")
@@ -440,7 +412,7 @@
         item = self.backend.create_item(u"test item metadata invalid change")
         item.change_metadata()
         item[u"change but"] = u"don't publish"
-        py.test.raises(NoSuchItemError, self.backend.get_item, "test item metadata invalid change")
+        pytest.raises(NoSuchItemError, self.backend.get_item, "test item metadata invalid change")
 
     def test_item_create_existing_mixed_1(self):
         item1 = self.backend.create_item(u'existing now 0')
@@ -448,7 +420,7 @@
         item2 = self.backend.create_item(u'existing now 0')
         item1.publish_metadata()
         item2.create_revision(0)
-        py.test.raises(ItemAlreadyExistsError, item2.commit)
+        pytest.raises(ItemAlreadyExistsError, item2.commit)
 
     def test_item_create_existing_mixed_2(self):
         item1 = self.backend.create_item(u'existing now 0')
@@ -456,7 +428,7 @@
         item2 = self.backend.create_item(u'existing now 0')
         item2.create_revision(0)
         item2.commit()
-        py.test.raises(ItemAlreadyExistsError, item1.publish_metadata)
+        pytest.raises(ItemAlreadyExistsError, item1.publish_metadata)
 
     def test_item_multiple_change_metadata_after_create(self):
         name = u"foo"
@@ -467,7 +439,7 @@
         item1[u"a"] = u"a"
         item2[u"a"] = u"b"
         item1.publish_metadata()
-        py.test.raises(ItemAlreadyExistsError, item2.publish_metadata)
+        pytest.raises(ItemAlreadyExistsError, item2.publish_metadata)
         item = self.backend.get_item(name)
         assert item[u"a"] == u"a"
 
@@ -483,12 +455,12 @@
     def test_metadata(self):
         self.create_rev_item_helper(u'no metadata')
         item = self.backend.get_item(u'no metadata')
-        py.test.raises(KeyError, item.__getitem__, u'asdf')
+        pytest.raises(KeyError, item.__getitem__, u'asdf')
 
     def test_revision(self):
         self.create_meta_item_helper(u'no revision')
         item = self.backend.get_item(u'no revision')
-        py.test.raises(NoSuchRevisionError, item.get_revision, -1)
+        pytest.raises(NoSuchRevisionError, item.get_revision, -1)
 
     def test_create_revision_change_meta(self):
         item = self.backend.create_item(u"double")
@@ -570,7 +542,7 @@
         item1.create_revision(1)
         item2.create_revision(1)
         item1.commit()
-        py.test.raises(RevisionAlreadyExistsError, item2.commit)
+        pytest.raises(RevisionAlreadyExistsError, item2.commit)
 
     def test_timestamp(self):
         tnow = int(time.time())
@@ -631,7 +603,7 @@
         assert not itemname in item_names
 
     def test_destroy_revision(self):
-        itemname = u"I will see my children die :-("
+        itemname = u"I will see my children die"        # removed the smiley ':-(' temporarily as it slows the test in addition with a failure
         rev_data = "I will die!"
         persistent_rev = "I will see my sibling die :-("
         item = self.backend.create_item(itemname)
--- a/MoinMoin/storage/_tests/test_backends_flatfile.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends_flatfile.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,9 +6,9 @@
 """
 
 
-import py
+import pytest
 
-py.test.skip("BackendTest base class tests quite some stuff that this very simple backend does not provide")
+pytest.skip("BackendTest base class tests quite some stuff that this very simple backend does not provide")
 # e.g.: revisioning, extremely long item names, metadata support
 # TODO: either fix base class so that it is more useful even to test simple backends,
 #       or implement some specific, more simple tests here.
--- a/MoinMoin/storage/_tests/test_backends_fs19.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends_fs19.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,7 +9,7 @@
 
 import os, re, tempfile, shutil
 
-import py.test
+import pytest
 
 from flask import current_app as app
 
@@ -97,8 +97,8 @@
         self.backend = None
 
     def test_get_item_that_doesnt_exist(self):
-        py.test.raises(NoSuchItemError, self.backend.get_item, "i_do_not_exist")
-        py.test.raises(NoSuchItemError, self.backend.get_item, item_name + "/not_exist.txt")
+        pytest.raises(NoSuchItemError, self.backend.get_item, "i_do_not_exist")
+        pytest.raises(NoSuchItemError, self.backend.get_item, item_name + "/not_exist.txt")
 
     def test_has_item_that_doesnt_exist(self):
         assert not self.backend.has_item("i_do_not_exist")
@@ -160,7 +160,7 @@
 
     def test_metadata_that_doesnt_exist(self):
         item = self.backend.get_item(item_name)
-        py.test.raises(KeyError, item.__getitem__, 'asdf')
+        pytest.raises(KeyError, item.__getitem__, 'asdf')
 
     def test_metadata_mtime(self):
         item = self.backend.get_item(item_name)
@@ -181,12 +181,12 @@
 
     def test_revision_that_doesnt_exist(self):
         item = self.backend.get_item(item_name)
-        py.test.raises(NoSuchRevisionError, item.get_revision, 42)
+        pytest.raises(NoSuchRevisionError, item.get_revision, 42)
 
     def test_revision_attachment_that_doesnt_exist(self):
         name = item_name + '/' + attachment_name
         item = self.backend.get_item(name)
-        py.test.raises(NoSuchRevisionError, item.get_revision, 1) # attachment only has rev 0
+        pytest.raises(NoSuchRevisionError, item.get_revision, 1) # attachment only has rev 0
 
     def test_revision_attachment_acl(self):
         name = deleted_item_name + '/' + attachment_name
@@ -276,5 +276,24 @@
             assert meta.get(TAGS, []) == expected_tags
             assert data == expected_data
 
+def test__decode_list():
+    from MoinMoin.storage.backends.fs19 import _decode_list
+    test_line = "test_item1 \t test_item2\n \t test_item3 \t"
+    result = _decode_list(test_line)
+    expected = ('test_item1', 'test_item2', 'test_item3')
+    assert result == expected
 
+def test__decode_dict():
+    from MoinMoin.storage.backends.fs19 import _decode_dict
+    test_line = "test_item1: first item\n \t test_item: second item2 \t \ntest_item3: third item \t"
+    result = _decode_dict(test_line)
+    expected = {'test_item1': ' first item', 'test_item3': ' third item', 'test_item': ' second item2'}
+    assert result == expected
 
+def test_hash_hexdigest():
+    from MoinMoin.storage.backends.fs19 import hash_hexdigest
+    result = hash_hexdigest('test_content')
+    assert result[0] == 12
+    with pytest.raises(ValueError):
+        hash_hexdigest(u'test_content')
+
--- a/MoinMoin/storage/_tests/test_backends_fs2.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends_fs2.py	Thu Sep 01 00:16:33 2011 +0200
@@ -17,7 +17,7 @@
 
     def create_backend(self):
         self.tempdir = tempfile.mkdtemp('', 'moin-')
-        return RouterBackend([('/', FS2Backend(self.tempdir))], cfg=app.cfg)
+        return FS2Backend(self.tempdir)
 
     def kill_backend(self):
         try:
--- a/MoinMoin/storage/_tests/test_backends_hg.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends_hg.py	Thu Sep 01 00:16:33 2011 +0200
@@ -11,19 +11,19 @@
 from tempfile import mkdtemp, mkstemp, gettempdir
 import shutil
 import os
-import py
+import pytest
 
 try:
     import mercurial
 except ImportError:
-    py.test.skip('Cannot test without Mercurial installed.')
+    pytest.skip('Cannot test without Mercurial installed.')
 
 from MoinMoin.storage._tests.test_backends import BackendTest
 from MoinMoin.storage.backends.hg import MercurialBackend
 from MoinMoin.storage.error import BackendError
 
 class TestMercurialBackend(BackendTest):
-
+    pytestmark = pytest.mark.xfail(reason='not maintained')
 
     def create_backend(self):
         self.test_dir = mkdtemp()
@@ -44,7 +44,7 @@
             assert isinstance(MercurialBackend(nonexisting_nested), MercurialBackend)
             assert isinstance(MercurialBackend(emptydir), MercurialBackend)
             assert isinstance(MercurialBackend(emptydir), MercurialBackend) # init on existing
-            py.test.raises(BackendError, MercurialBackend, file)
+            pytest.raises(BackendError, MercurialBackend, file)
             assert isinstance(MercurialBackend(dirstruct), MercurialBackend)
         finally:
             shutil.rmtree(emptydir)
@@ -55,9 +55,9 @@
     def test_permission(self):
         import sys
         if sys.platform == 'win32':
-            py.test.skip("Not much usable test on win32.")
+            pytest.skip("Not much usable test on win32.")
         no_perms = os.path.join("/", "permission-error-dir")
-        py.test.raises(BackendError, MercurialBackend, no_perms)
+        pytest.raises(BackendError, MercurialBackend, no_perms)
 
     def test_backend_init_non_empty_datadir(self):
         datadir = mkdtemp()
--- a/MoinMoin/storage/_tests/test_backends_memory.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends_memory.py	Thu Sep 01 00:16:33 2011 +0200
@@ -25,7 +25,6 @@
         pass
 
 class TestTracingBackend(BackendTest):
-
     def create_backend(self):
         import random
         return TracingBackend()#"/tmp/codebuf%i.py" % random.randint(1, 2**30))
--- a/MoinMoin/storage/_tests/test_backends_router.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends_router.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,15 +7,21 @@
     This defines tests for the RouterBackend
 """
 
+import os
+import time
 
-import py
+import pytest
 
 from flask import current_app as app
 
+from whoosh.query import Term, And, Every
+
+from MoinMoin.config import NAME, MTIME
 from MoinMoin.error import ConfigurationError
 from MoinMoin.storage._tests.test_backends import BackendTest
 from MoinMoin.storage.backends.memory import MemoryBackend
 from MoinMoin.storage.backends.router import RouterBackend
+from MoinMoin.search.indexing import WhooshIndex
 
 class TestRouterBackend(BackendTest):
     """
@@ -34,6 +40,16 @@
     def kill_backend(self):
         pass
 
+    def teardown_method(self, method):
+        # clean the index directory after each test as messes with the backend history
+        # XXX tests with backend.history should not be failing due to contents in index directory
+        # the contents of the directory and the way backend.history is handled should be implemented
+        # in a better way
+        index_dir = WhooshIndex()._index_dir
+        for values in os.walk(index_dir):
+            for index_file_name in values[2]:
+                index_file = index_dir + '/' + index_file_name
+                os.remove(index_file)
 
     def test_correct_backend(self):
         mymap = {u'rootitem': self.root,         # == /rootitem
@@ -52,13 +68,15 @@
         itemname = u'child/foo'
         item = self.backend.create_item(itemname)
         assert item.name == itemname
-        assert item._backend is self.child
+        # using item._backend to get the backend makes this test fail.
+        test_backend, child_name, root_name = item._get_backend(itemname)
+        assert test_backend is self.child
         item.change_metadata()
         item[u'just'] = u'testing'
         item.publish_metadata()
-
-        item = self.backend.get_item(itemname)
-        assert item._backend is self.child
+        # using item._backend to get the backend makes this test fail.
+        test_backend, child_name, root_name = item._get_backend(itemname)
+        assert test_backend is self.child
         assert item[u'just'] == u'testing'
         assert item.name == itemname
 
@@ -130,102 +148,41 @@
         assert name == ''
         assert mountpoint == 'child'
 
-
-    def test_history(self):
-        order = [(u'first', 0, ), (u'second', 0, ), (u'first', 1, ), (u'a', 0), (u'child/my_subitem', 0) ]
-        for name, revno in order:
-            if revno == 0:
-                item = self.backend.create_item(name)
-            else:
-                item = self.backend.get_item(name)
-            item.create_revision(revno)
+    def test_search_item_history_order(self):
+        item_name = u'some item'
+        item = self.backend.create_item(item_name)
+        for rev_no in range(3):
+            rev = item.create_revision(rev_no)
             item.commit()
-
-            # Revisions are created too fast for the rev's timestamp's granularity.
-            # This only affects the RouterBackend because there several different
-            # backends are used and no means for storing simultaneously created revs
-            # in the correct order exists between backends. It affects AclWrapperBackend
-            # tests as well because those use a RouterBackend internally for real-world-likeness.
-
-            # XXX XXX
-            # You may have realized that all the items above belong to the same backend so this shouldn't actually matter.
-            # It does matter, however, once you consider that the RouterBackend uses the generic, slow history implementation.
-            # This one uses iteritems and then sorts all the revisions itself, hence discarding any information of ordering
-            # for simultaneously created revisions. If we just call history of that single backend directly, it works without
-            # time.sleep. For n backends, however, you'd have to somehow merge the revisions into one generator again, thus
-            # discarding that information again. Besides, that would be a costly operation. The ordering for simultaneosly
-            # created revisions remains the same since it's based on tuple ordering. Better proposals welcome.
-            import time
-            time.sleep(1)
-
-        for num, rev in enumerate(self.backend.history(reverse=False)):
-            name, revno = order[num]
-            assert rev.item.name == name
-            assert rev.revno == revno
-
-        order.reverse()
-        for num, rev in enumerate(self.backend.history(reverse=True)):
-            name, revno = order[num]
-            assert rev.item.name == name
-            assert rev.revno == revno
+        query = Term("name_exact", item_name)
+        results = list(self.backend.search(query, all_revs=True, sortedby="rev_no"))
+        print results
+        assert results[0].get("rev_no") == 0
+        assert results[1].get("rev_no") == 1
+        assert results[2].get("rev_no") == 2
+        results = list(self.backend.search(query, all_revs=True, sortedby="rev_no", reverse=True))
+        print results
+        assert results[0].get("rev_no") == 2
+        assert results[1].get("rev_no") == 1
+        assert results[2].get("rev_no") == 0
 
-    # See history function in indexing.py for comments on why this test fails.
-    @py.test.mark.xfail
-    def test_history_size_after_rename(self):
-        item = self.backend.create_item(u'first')
-        item.create_revision(0)
-        item.commit()
-        item.rename(u'second')
-        item.create_revision(1)
-        item.commit()
-        assert len([rev for rev in self.backend.history()]) == 2
-
-    def test_history_after_destroy_item(self):
-        itemname = u"I will be completely destroyed"
-        rev_data = "I will be completely destroyed, too, hopefully"
-        item = self.backend.create_item(itemname)
-        rev = item.create_revision(0)
-        rev.write(rev_data)
-        item.commit()
-
-        item.destroy()
-
-        all_rev_data = [rev.read() for rev in self.backend.history()]
-        assert not rev_data in all_rev_data
-
-        for rev in self.backend.history():
-            assert not rev.item.name == itemname
-        for rev in self.backend.history(reverse=False):
-            assert not rev.item.name == itemname
+    def test_search_global_history_order(self):
+        names = [u'foo', u'bar', u'baz', ]
+        for item_name in names:
+            item = self.backend.create_item(item_name)
+            rev = item.create_revision(0)
+            item.commit()
+            time.sleep(1) # make sure we have different MTIME
+        query = Every()
+        results = list(self.backend.search(query, all_revs=True, sortedby=[MTIME, "rev_no"]))
+        print results
+        assert results[0].get(NAME) == names[0]
+        assert results[1].get(NAME) == names[1]
+        assert results[2].get(NAME) == names[2]
+        results = list(self.backend.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True))
+        print results
+        assert results[0].get(NAME) == names[2]
+        assert results[1].get(NAME) == names[1]
+        assert results[2].get(NAME) == names[0]
 
-    def test_history_after_destroy_revision(self):
-        itemname = u"I will see my children die :-("
-        rev_data = "I will die!"
-        persistent_rev = "I will see my sibling die :-("
-        item = self.backend.create_item(itemname)
-        rev = item.create_revision(0)
-        rev.write(rev_data)
-        item.commit()
-        rev = item.create_revision(1)
-        rev.write(persistent_rev)
-        item.commit()
-
-        rev = item.get_revision(0)
-        rev.destroy()
 
-        for rev in self.backend.history():
-            assert not (rev.item.name == itemname and rev.revno == 0)
-
-    def test_history_item_names(self):
-        item = self.backend.create_item(u'first')
-        item.create_revision(0)
-        item.commit()
-        item.rename(u'second')
-        item.create_revision(1)
-        item.commit()
-        revs_in_create_order = [rev for rev in self.backend.history(reverse=False)]
-        assert revs_in_create_order[0].revno == 0
-        assert revs_in_create_order[0].item.name == u'second'
-        assert revs_in_create_order[1].revno == 1
-        assert revs_in_create_order[1].item.name == u'second'
-
--- a/MoinMoin/storage/_tests/test_backends_sqla.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_backends_sqla.py	Thu Sep 01 00:16:33 2011 +0200
@@ -14,11 +14,16 @@
 
 from MoinMoin.storage._tests.test_backends import BackendTest
 from MoinMoin.storage.backends.sqla import SQLAlchemyBackend, SQLARevision, Data
-
+from MoinMoin.search.indexing import WhooshIndex
 
 class TestSQLABackend(BackendTest):
 
     def create_backend(self):
+        # when running py.test, all the index files are removed in index dir(please see teardown_method in test_backends_router)
+        # initializing WhooshIndex creates all_revisions_index and latest_revisions_index in there.
+        # without index files item.commit() raises EmptyIndexError
+        # SQLAlchemyBackend do not initializes revision files there
+        WhooshIndex()
         return SQLAlchemyBackend(verbose=True)
 
     def kill_backend(self):
--- a/MoinMoin/storage/_tests/test_indexing.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_indexing.py	Thu Sep 01 00:16:33 2011 +0200
@@ -13,35 +13,34 @@
 from MoinMoin.config import NAME
 
 # Revisions for tests
-document_revs = [
-                 { "wikiname": u"Test",
-                   "name": u"DocumentOne",
-                   "uuid": u"68054804bd7141609b7c441143adf83d",
-                   "rev_no": 0,
-                   "mtime":  1172969203.1,
-                   "content": u"Some not very long content line",
-                   "contenttype": u"text/plain;charset=utf-8",
-                   "tags": [u"Rest", u"in", u"peace"],
-                   "itemlinks": [u"Home", u"Find"],
-                   "itemtransclusions": [u"Another", u"Stuff"],
-                   "language": u"en",
-                   "address": u"127.0.0.1",
-                   "hostname": u"localhost",
+document_revs = [{"wikiname": u"Test",
+                  "name": u"DocumentOne",
+                  "uuid": u"68054804bd7141609b7c441143adf83d",
+                  "rev_no": 0,
+                  "mtime":  1172969203.1,
+                  "content": u"Some not very long content line",
+                  "contenttype": u"text/plain;charset=utf-8",
+                  "tags": [u"Rest", u"in", u"peace"],
+                  "itemlinks": [u"Home", u"Find"],
+                  "itemtransclusions": [u"Another", u"Stuff"],
+                  "language": u"en",
+                  "address": u"127.0.0.1",
+                  "hostname": u"localhost",
                  },
-                 { "wikiname": u"Test",
-                   "name": u"DocumentOne",
-                   "uuid": u"68054804bd7141609b7c441143adf83d",
-                   "rev_no": 1,
-                   "mtime":  1172969203.9,
-                   "content": u"This line should be much better, but it isn't",
-                   "contenttype": u"text/plain;charset=utf-8",
-                   "tags": [u"first_tag", u"second_tag"],
-                   "itemlinks": [u"Home", u"Find"],
-                   "itemtransclusions": [u"Another", u"Stuff"],
-                   "language": u"en",
-                   "address": u"127.0.0.1",
-                   "hostname": u"localhost",
-                 }
+                 {"wikiname": u"Test",
+                  "name": u"DocumentOne",
+                  "uuid": u"68054804bd7141609b7c441143adf83d",
+                  "rev_no": 1,
+                  "mtime":  1172969203.9,
+                  "content": u"This line should be much better, but it isn't",
+                  "contenttype": u"text/plain;charset=utf-8",
+                  "tags": [u"first_tag", u"second_tag"],
+                  "itemlinks": [u"Home", u"Find"],
+                  "itemtransclusions": [u"Another", u"Stuff"],
+                  "language": u"en",
+                  "address": u"127.0.0.1",
+                  "hostname": u"localhost",
+                 },
                 ]
 
 class TestIndexing(object):
@@ -53,7 +52,7 @@
         self.latest_revs_ix = self.item_index.index_object.latest_revisions_index
 
     def teardown_method(self, method):
-         self.item_index.remove_index()
+        self.item_index.remove_index()
 
     def test_create_item(self):
         """ Try to search for non-existent revision, add it to backend and then search again """
--- a/MoinMoin/storage/_tests/test_middleware_acl.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_middleware_acl.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,7 +8,7 @@
 """
 
 
-import py
+import pytest
 
 from flask import g as flaskg
 
@@ -46,7 +46,7 @@
     def test_noaccess(self):
         name = u"noaccess"
         self.create_item_acl(name, u"All:")
-        assert py.test.raises(AccessDeniedError, self.get_item, name)
+        assert pytest.raises(AccessDeniedError, self.get_item, name)
 
     def test_create_item(self):
         class Config(wikiconfig.Config):
@@ -54,7 +54,7 @@
             content_acl = dict(default=u"All:admin,read,write,destroy")
 
         backend = flaskg.storage
-        assert py.test.raises(AccessDeniedError, backend.create_item, u"I will never exist")
+        assert pytest.raises(AccessDeniedError, backend.create_item, u"I will never exist")
 
         item = self.create_item_acl(u"i will exist!", u"All:read,write")
         rev = item.create_revision(1)
@@ -70,13 +70,13 @@
         item = self.get_item(name)
 
         # Should not...
-        assert py.test.raises(AccessDeniedError, item.create_revision, 1)
-        assert py.test.raises(AccessDeniedError, item.change_metadata)
+        assert pytest.raises(AccessDeniedError, item.create_revision, 1)
+        assert pytest.raises(AccessDeniedError, item.change_metadata)
 
     def test_write_after_create(self):
         name = u"writeaftercreate"
         item = self.create_item_acl(name, u"All:")
-        assert py.test.raises(AccessDeniedError, item.create_revision, 1)
+        assert pytest.raises(AccessDeniedError, item.create_revision, 1)
 
     def test_modify_without_acl_change(self):
         name = u"copy_without_acl_change"
@@ -95,7 +95,7 @@
         item = self.get_item(name)
         rev = item.create_revision(1)
         # without admin rights it is disallowed to change ACL
-        py.test.raises(AccessDeniedError, rev.__setitem__, ACL, acl + u",destroy")
+        pytest.raises(AccessDeniedError, rev.__setitem__, ACL, acl + u",destroy")
 
     def test_write_without_read(self):
         name = u"write_but_not_read"
@@ -106,6 +106,6 @@
         rev.write("My name is " + name)
         item.commit()
 
-        py.test.raises(AccessDeniedError, item.get_revision, -1)
-        py.test.raises(AccessDeniedError, item.get_revision, 0)
+        pytest.raises(AccessDeniedError, item.get_revision, -1)
+        pytest.raises(AccessDeniedError, item.get_revision, 0)
 
--- a/MoinMoin/storage/_tests/test_serialization.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/test_serialization.py	Thu Sep 01 00:16:33 2011 +0200
@@ -10,8 +10,6 @@
 """
 
 
-import py
-
 from StringIO import StringIO
 
 from flask import g as flaskg
--- a/MoinMoin/storage/_tests/test_terms.py	Wed Aug 24 11:29:29 2011 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,290 +0,0 @@
-# Copyright: 2008 MoinMoin:JohannesBerg
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-    MoinMoin - Term tests.
-"""
-
-
-import re
-
-from MoinMoin.config import NAME, CONTENTTYPE
-from MoinMoin.storage import terms as term
-from MoinMoin.storage.backends.memory import MemoryBackend
-
-
-_item_contents = {
-    u'a': u'abcdefg hijklmnop',
-    u'b': u'bbbbbbb bbbbbbbbb',
-    u'c': u'Abiturienten Apfeltortor',
-    u'Lorem': u'Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Duis placerat, tortor quis sollicitudin dictum, nisi tellus aliquam quam, ac varius lacus diam eget tortor. Nulla vehicula, nisi ac hendrerit aliquam, libero erat tempor ante, lobortis placerat lacus justo vitae erat. In rutrum odio a sem. In ac risus vel diam vulputate luctus. Fusce sit amet est. Morbi consectetuer eros vel risus. In nulla lacus, ultrices id, vestibulum tempus, dictum in, mauris. Quisque rutrum faucibus nisl. Suspendisse potenti. In hac habitasse platea dictumst. Donec ac magna ac eros malesuada facilisis. Pellentesque viverra nibh nec dui. Praesent venenatis lectus vehicula eros. Phasellus pretium, ante at mollis luctus, nibh lacus ultricies eros, vitae pharetra lacus leo at neque. Nullam vel sapien. In in diam id massa nonummy suscipit. Curabitur vel dui sed tellus pellentesque pretium.',
-}
-
-_item_metadata = {
-    u'a': {'m1': 'True', 'm2': '222'},
-    u'A': {'m1': 'True', 'm2': '333'},
-    u'b': {'m1': 'False', 'm2': '222'},
-    u'c': {'m1': 'True', 'm2': '222'},
-    u'B': {'m1': 'False', 'm2': '333'},
-    u'Lorem': {'m1': '7', 'm2': '444'},
-}
-
-_lastrevision_metadata = {
-    u'a': {'a': '1'},
-    u'A': {'a': ''},
-    u'b': {'a': '0'},
-    u'c': {'a': 'False'},
-    u'B': {'a': ''},
-    u'Lorem': {'a': '42'},
-}
-
-for n in _item_contents.keys():
-    nl = n.lower()
-    nu = n.upper()
-    _item_contents[nl] = _item_contents[n].lower()
-    _item_contents[nu] = _item_contents[n].upper()
-    if not nl in _item_metadata:
-        _item_metadata[nl] = _item_metadata[n]
-    if not nu in _item_metadata:
-        _item_metadata[nu] = _item_metadata[n]
-    if not nl in _lastrevision_metadata:
-        _lastrevision_metadata[nl] = _lastrevision_metadata[n]
-    if not nu in _lastrevision_metadata:
-        _lastrevision_metadata[nu] = _lastrevision_metadata[n]
-
-memb = MemoryBackend()
-for iname, md in _item_metadata.iteritems():
-    item = memb.create_item(iname)
-    item.change_metadata()
-    item.update(md)
-    item.publish_metadata()
-
-    rev = item.create_revision(0)
-    md = _lastrevision_metadata[iname]
-    rev.update({NAME: iname, CONTENTTYPE: u"application/octet-stream"})
-    rev.update(md)
-    rev.write(_item_contents[iname])
-    item.commit()
-
-item = memb.create_item('NR')
-item.change_metadata()
-item.update({'m1': 'True'})
-item.publish_metadata()
-del item
-
-class TermTestData:
-    def __init__(self, text):
-        self.text = text
-    def read(self, size=None):
-        return self.text
-
-class CacheAssertTerm(term.Term):
-    def __init__(self):
-        term.Term.__init__(self)
-        self.evalonce = False
-
-    def _evaluate(self, item):
-        assert not self.evalonce
-        self.evalonce = True
-        return True
-
-class AssertNotCalledTerm(term.Term):
-    def _evaluate(self, item):
-        assert False
-
-class TestTerms:
-    def _evaluate(self, term, itemname, expected):
-        if itemname is not None:
-            item = memb.get_item(itemname)
-        else:
-            item = None
-        term.prepare()
-        assert expected == term.evaluate(item)
-
-    def testSimpleTextSearch(self):
-        terms = [term.Text(u'abcdefg', True), term.Text(u'ijklmn', True)]
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testSimpleTextSearchCI(self):
-        terms = [term.Text(u'abcdefg', False), term.Text(u'ijklmn', False)]
-        for item, expected in [('a', True), ('A', True), ('b', False), ('B', False), ('lorem', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testANDOR(self):
-        tests = [
-            (True,  [1, 1, 1, 1, 1]),
-            (True,  [1, 1, 1, 1]),
-            (True,  [1, 1, 1]),
-            (True,  [1, 1]),
-            (False, [0, 1, 1]),
-            (False, [0, 1, 1, 1]),
-            (False, [1, 0, 1, 1]),
-            (False, [1, 1, 0, 1]),
-            (False, [1, 1, 1, 0]),
-            (False, [0, 1, 1, 0]),
-        ]
-        for expected, l in tests:
-            l = [term.BOOL(i) for i in l]
-            t = term.AND(*l)
-            yield self._evaluate, t, 'a', expected
-        for expected, l in tests:
-            l = [term.BOOL(1 - i) for i in l]
-            t = term.OR(*l)
-            yield self._evaluate, t, 'a', not expected
-
-    def testXOR(self):
-        tests = [
-            (False, [1, 1, 1, 1, 1]),
-            (False, [1, 1, 1, 1]),
-            (False, [1, 1, 1]),
-            (False, [1, 1]),
-            (False, [0, 1, 1]),
-            (False, [0, 1, 1, 1]),
-            (False, [1, 0, 1, 1]),
-            (False, [1, 1, 0, 1]),
-            (False, [1, 1, 1, 0]),
-            (False, [0, 1, 1, 0]),
-            (True,  [0, 0, 0, 1, 0]),
-            (True,  [0, 0, 1, 0]),
-            (True,  [1, 0, 0]),
-            (True,  [0, 1]),
-            (False, [0, 0, 0]),
-        ]
-        for expected, l in tests:
-            l = [term.BOOL(i) for i in l]
-            t = term.XOR(*l)
-            yield self._evaluate, t, 'a', expected
-
-    def testTextSearchRE(self):
-        terms = [term.TextRE(re.compile('^abc')), term.TextRE(re.compile('\shij'))]
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testTextSearchRE2(self):
-        terms = [term.TextRE(re.compile('sollici')), term.TextRE(re.compile('susci'))]
-        for item, expected in [('a', False), ('A', False), ('b', False), ('B', False), ('lorem', True), ('NR', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testResultCaching1(self):
-        cat = CacheAssertTerm()
-        expected = True
-        t = term.AND(cat, cat, cat)
-        yield self._evaluate, t, None, expected
-
-    def testResultCaching2(self):
-        cat = CacheAssertTerm()
-        expected = True
-        t = term.OR(cat, cat, cat)
-        yield self._evaluate, t, None, expected
-
-    def testResultCaching3(self):
-        cat = CacheAssertTerm()
-        expected = False
-        t = term.AND(cat, cat, cat, term.FALSE)
-        yield self._evaluate, t, None, expected
-
-    def testResultCaching4(self):
-        cat = CacheAssertTerm()
-        expected = True
-        t = term.OR(cat, cat, cat)
-        yield self._evaluate, t, None, expected
-
-    def testShortCircuitEval1(self):
-        yield self._evaluate, term.AND(term.TRUE, term.FALSE, AssertNotCalledTerm()), None, False
-
-    def testShortCircuitEval2(self):
-        yield self._evaluate, term.OR(term.TRUE, term.FALSE, AssertNotCalledTerm()), None, True
-
-    def testSimpleTitleSearch(self):
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, term.Name(u'a', True), item, expected
-
-    def testSimpleTitleSearchCI(self):
-        for item, expected in [('a', True), ('A', True), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, term.Name(u'a', False), item, expected
-
-    def testTitleRESearch(self):
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, term.NameRE(re.compile('(a|e)')), item, expected
-
-    def testMetaMatch1(self):
-        t = term.ItemMetaDataMatch('m1', 'True')
-        for item, expected in [('a', True), ('A', True), ('b', False), ('B', False), ('lorem', False), ('NR', True)]:
-            yield self._evaluate, t, item, expected
-
-    def testMetaMatch2(self):
-        t = term.ItemMetaDataMatch('m2', '333')
-        for item, expected in [('a', False), ('A', True), ('b', False), ('B', True), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testMetaMatch3(self):
-        t = term.ItemMetaDataMatch('m2', '444')
-        for item, expected in [('a', False), ('A', False), ('b', False), ('B', False), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta1(self):
-        t = term.ItemHasMetaDataKey('m3')
-        for item, expected in [('a', False), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta2(self):
-        t = term.ItemHasMetaDataKey('m1')
-        for item, expected in [('a', True), ('A', True), ('b', True), ('B', True), ('lorem', True), ('NR', True)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta3(self):
-        t = term.LastRevisionHasMetaDataKey('a')
-        for item, expected in [('a', True), ('A', True), ('b', True), ('B', True), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta4(self):
-        t = term.LastRevisionMetaDataMatch('a', '')
-        for item, expected in [('a', False), ('A', True), ('b', False), ('B', True), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testNameFn(self):
-        t = term.NameFn(lambda x: x in ['a', 'b', 'lorem'])
-        for item, expected in [('a', True), ('A', False), ('b', True), ('B', False), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordCI(self):
-        t = term.Word('Curabitur', False)
-        for item, expected in [('B', False), ('Lorem', True), ('lorem', True), ('LOREM', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWord(self):
-        t = term.Word('Curabitur', True)
-        for item, expected in [('B', False), ('Lorem', True), ('lorem', False), ('LOREM', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStartCI(self):
-        t = term.WordStart('Curabi', False)
-        for item, expected in [('B', False), ('Lorem', True), ('lorem', True), ('LOREM', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStart(self):
-        t = term.WordStart('Curabi', True)
-        for item, expected in [('c', False), ('Lorem', True), ('lorem', False), ('LOREM', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStart2(self):
-        t = term.WordStart('abitur', True)
-        for item, expected in [('c', True), ('C', False), ('Lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStart2CI(self):
-        t = term.WordStart('abitur', False)
-        for item, expected in [('c', True), ('C', True), ('Lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordEndCI(self):
-        t = term.WordEnd('abitur', False)
-        for item, expected in [('c', False), ('Lorem', True), ('lorem', True), ('LOREM', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-coverage_modules = ['MoinMoin.storage.terms']
--- a/MoinMoin/storage/_tests/tests_backend_api.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/_tests/tests_backend_api.py	Thu Sep 01 00:16:33 2011 +0200
@@ -5,7 +5,7 @@
     MoinMoin - Test - storage API
 """
 
-import py
+import pytest
 
 from MoinMoin.storage import Backend, Item, StoredRevision, NewRevision
 from MoinMoin.storage.error import NoSuchItemError
@@ -65,7 +65,7 @@
         assert not oldrev.keys()
 
         newrev = item.create_revision(1)
-        py.test.raises(TypeError, newrev.__setitem__, '__reserved')
+        pytest.raises(TypeError, newrev.__setitem__, '__reserved')
 
         assert not newrev.keys()
 
--- a/MoinMoin/storage/backends/acl.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/backends/acl.py	Thu Sep 01 00:16:33 2011 +0200
@@ -95,17 +95,6 @@
         # up on the real backend.
         return getattr(self.backend, attr)
 
-    def search_items(self, searchterm):
-        """
-        @see: Backend.search_items.__doc__
-        """
-        for item in self.backend.search_items(searchterm):
-            if self._may(item.name, READ):
-                # The item returned needs to be wrapped because otherwise the
-                # item's methods (like create_revision) wouldn't be wrapped.
-                wrapped_item = AclWrapperItem(item, self)
-                yield wrapped_item
-
     def get_item(self, itemname):
         """
         @see: Backend.get_item.__doc__
--- a/MoinMoin/storage/backends/fileserver.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/backends/fileserver.py	Thu Sep 01 00:16:33 2011 +0200
@@ -1,14 +1,14 @@
-# Copyright: 2008-2010 MoinMoin:ThomasWaldmann
+# Copyright: 2008-2011 MoinMoin:ThomasWaldmann
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-    MoinMoin - file server backend
+MoinMoin - file server backend
 
-    You can use this backend to directly get read-only access to your
-    wiki server's filesystem.
+You can use this backend to directly get read-only access to your
+wiki server's filesystem.
 
-    TODO: nearly working, but needs more work at other places,
-          e.g. in the router backend, to be useful.
+TODO: nearly working, but needs more work at other places,
+      e.g. in the router backend, to be useful.
 """
 
 
@@ -24,7 +24,7 @@
 from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError
 from MoinMoin.util.mimetype import MimeType
 
-from MoinMoin.config import ACL, CONTENTTYPE, ACTION, COMMENT, MTIME, SIZE
+from MoinMoin.config import NAME, ACL, CONTENTTYPE, ACTION, COMMENT, MTIME, SIZE, HASH_ALGORITHM
 
 class FSError(Exception):
     """ file serving backend error """
@@ -65,7 +65,12 @@
 
     def iter_items_noindex(self):
         for dirpath, dirnames, filenames in os.walk(self.root_dir):
-            yield DirItem(self, self._path2item(dirpath))
+            name = self._path2item(dirpath)
+            if name:
+                # XXX currently there is an issue with whoosh indexing if fileserver
+                # backend is mounted at / and the item name is empty, resulting in a
+                # completely empty item name - avoid this for now.
+                yield DirItem(self, name)
             for filename in filenames:
                 try:
                     item = FileItem(self, self._path2item(os.path.join(dirpath, filename)))
@@ -160,9 +165,11 @@
         filepath = item._fs_filepath
         st = item._fs_stat
         meta = { # make something up
+            NAME: item.name,
             MTIME: int(st.st_mtime),
-            ACTION: 'SAVE',
+            ACTION: u'SAVE',
             SIZE: st.st_size,
+            HASH_ALGORITHM: u'' # XXX fake it, send_file needs it for etag and crashes ithout the hash
         }
         self._fs_meta = meta
         self._fs_data_fname = filepath
@@ -173,7 +180,7 @@
     def __init__(self, item, revno):
         FileDirRevision.__init__(self, item, revno)
         self._fs_meta.update({
-            CONTENTTYPE: 'text/x.moin.wiki',
+            CONTENTTYPE: u'text/x.moin.wiki;charset=utf-8',
         })
         # create a directory "page" in wiki markup:
         try:
@@ -203,6 +210,6 @@
         FileDirRevision.__init__(self, item, revno)
         contenttype = MimeType(filename=self._fs_data_fname).content_type()
         self._fs_meta.update({
-            CONTENTTYPE: contenttype,
+            CONTENTTYPE: unicode(contenttype),
         })
 
--- a/MoinMoin/storage/backends/fs19.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/backends/fs19.py	Thu Sep 01 00:16:33 2011 +0200
@@ -26,7 +26,11 @@
 UUID_LEN = len(make_uuid())
 
 from sqlalchemy import create_engine, MetaData, Table, Column, String, Unicode, Integer
-from sqlalchemy.exc import IntegrityError
+
+try:
+    from sqlalchemy.exc import IntegrityError
+except ImportError:
+    from sqlalchemy.exceptions import IntegrityError
 
 from MoinMoin import log
 logging = log.getLogger(__name__)
--- a/MoinMoin/storage/backends/fs2.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/backends/fs2.py	Thu Sep 01 00:16:33 2011 +0200
@@ -21,7 +21,12 @@
 from flask import current_app as app
 
 from sqlalchemy import create_engine, MetaData, Table, Column, String, Unicode, Integer
-from sqlalchemy.exc import IntegrityError
+
+try:
+    from sqlalchemy.exc import IntegrityError
+except ImportError:
+    from sqlalchemy.exceptions import IntegrityError
+
 from sqlalchemy.pool import NullPool
 
 from werkzeug import cached_property
--- a/MoinMoin/storage/backends/indexing.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/backends/indexing.py	Thu Sep 01 00:16:33 2011 +0200
@@ -22,15 +22,79 @@
 from uuid import uuid4
 make_uuid = lambda: unicode(uuid4().hex)
 
+from flask import current_app as app
+from flask import g as flaskg
+from flask import request
+
 from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError, \
                                    AccessDeniedError
-from MoinMoin.config import ACL, CONTENTTYPE, UUID, NAME, NAME_OLD, MTIME, TAGS
+from MoinMoin.config import ACL, CONTENTTYPE, UUID, NAME, NAME_OLD, MTIME, TAGS, \
+                            ADDRESS, HOSTNAME, USERID, ITEMLINKS, ITEMTRANSCLUSIONS
 from MoinMoin.search.indexing import backend_to_index
-from MoinMoin.converter import convert_to_indexable
+from MoinMoin.converter import default_registry
+from MoinMoin.util.iri import Iri
+from MoinMoin.util.mime import Type, type_moin_document
+from MoinMoin.util.tree import moin_page
+from MoinMoin import wikiutil
 
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
+
+def convert_to_indexable(rev, new_rev=False):
+    """
+    convert a revision to an indexable document
+
+    :param rev: item revision - please make sure that the content file is
+                ready to read all indexable content from it. if you have just
+                written that content or already read from it, you need to call
+                rev.seek(0) before calling convert_to_indexable(rev).
+    """
+    try:
+        # TODO use different converter mode?
+        # Maybe we want some special mode for the input converters so they emit
+        # different output than for normal rendering), esp. for the non-markup
+        # content types (images, etc.).
+        input_contenttype = rev[CONTENTTYPE]
+        output_contenttype = 'text/plain'
+        type_input_contenttype = Type(input_contenttype)
+        type_output_contenttype = Type(output_contenttype)
+        reg = default_registry
+        # first try a direct conversion (this could be useful for extraction
+        # of (meta)data from binary types, like from images or audio):
+        conv = reg.get(type_input_contenttype, type_output_contenttype)
+        if conv:
+            doc = conv(rev, input_contenttype)
+            return doc
+        # otherwise try via DOM as intermediate format (this is useful if
+        # input type is markup, to get rid of the markup):
+        input_conv = reg.get(type_input_contenttype, type_moin_document)
+        refs_conv = reg.get(type_moin_document, type_moin_document, items='refs')
+        output_conv = reg.get(type_moin_document, type_output_contenttype)
+        if input_conv and output_conv:
+            doc = input_conv(rev, input_contenttype)
+            # We do not convert smileys, includes, macros, links, because
+            # it does not improve search results or even makes results worse.
+            # We do run the referenced converter, though, to extract links and
+            # transclusions.
+            if new_rev:
+                # we only can modify new, uncommitted revisions, not stored revs
+                i = Iri(scheme='wiki', authority='', path='/' + rev[NAME])
+                doc.set(moin_page.page_href, unicode(i))
+                refs_conv(doc)
+                # side effect: we update some metadata:
+                rev[ITEMLINKS] = refs_conv.get_links()
+                rev[ITEMTRANSCLUSIONS] = refs_conv.get_transclusions()
+            doc = output_conv(doc)
+            return doc
+        # no way
+        raise TypeError("No converter for %s --> %s" % (input_contenttype, output_contenttype))
+    except Exception as e: # catch all exceptions, we don't want to break an indexing run
+        logging.exception("Exception happened in conversion of item %r rev %d contenttype %s:" % (rev[NAME], rev.revno, rev[CONTENTTYPE]))
+        doc = u'ERROR [%s]' % str(e)
+        return doc
+
+
 class IndexingBackendMixin(object):
     """
     Backend indexing support / functionality using the index.
@@ -57,43 +121,20 @@
         item.publish_metadata()
         return item
 
-    def history(self, reverse=True, item_name=u'', start=None, end=None):
-        """
-        History implementation using the index.
-        """
-        for result in self._index.history(reverse=reverse, item_name=item_name, start=start, end=end):
-            # we currently create the item, the revision and yield it to stay
-            # compatible with storage api definition, but this could be changed to
-            # just return the data we get from the index (without accessing backend)
-            # TODO: A problem exists at item = self.get_item(name).
-            # In the history_size_after_rename test in test_backends.py,
-            # an item was created with the name "first" and then renamed to "second."
-            # When it runs through this history function and runs item = self.get_item("first"),
-            # it can't find it because it was already renamed to "second."
-            # Some suggested solutions are: using some neverchanging uuid to identify some specific item
-            # or continuing to use the name, but tracking name changes within the item's history.
-            rev_datetime, name, rev_no = result
-            try:
-                logging.debug("HISTORY: name %s revno %s" % (name, rev_no))
-                item = self.get_item(name)
-                yield item.get_revision(rev_no)
-            except AccessDeniedError as e:
-                # just skip items we may not access
-                pass
-            except (NoSuchItemError, NoSuchRevisionError) as e:
-                logging.exception("history processing catched exception")
+    def query_parser(self, default_fields, all_revs=False):
+        return self._index.query_parser(default_fields, all_revs=all_revs)
 
-    def all_tags(self):
-        """
-        Return a unsorted list of tuples (count, tag, tagged_itemnames) for all tags.
-        """
-        return self._index.all_tags()
+    def searcher(self, all_revs=False):
+        return self._index.searcher(all_revs=all_revs)
 
-    def tagged_items(self, tag):
-        """
-        Return a list of item names of items that are tagged with <tag>.
-        """
-        return self._index.tagged_items(tag)
+    def search(self, q, all_revs=False, **kw):
+        return self._index.search(q, all_revs=all_revs, **kw)
+
+    def search_page(self, q, all_revs=False, pagenum=1, pagelen=10, **kw):
+        return self._index.search_page(q, all_revs=all_revs, pagenum=pagenum, pagelen=pagelen, **kw)
+
+    def documents(self, all_revs=False, **kw):
+        return self._index.documents(all_revs=all_revs, **kw)
 
 
 class IndexingItemMixin(object):
@@ -141,8 +182,9 @@
         """
         update the index, removing everything related to this item
         """
-        logging.debug("item %r remove index!" % (self.name, ))
-        self._index.remove_item(metas=self)
+        uuid = self[UUID]
+        logging.debug("item %r %r remove index!" % (self.name, uuid))
+        self._index.remove_item(uuid)
 
 
 class IndexingRevisionMixin(object):
@@ -166,6 +208,7 @@
         name = self.item.name
         uuid = self.item[UUID]
         revno = self.revno
+        logging.debug("Processing: name %s revno %s" % (name, revno))
         if MTIME not in self:
             self[MTIME] = int(time.time())
         if NAME not in self:
@@ -174,11 +217,29 @@
             self[UUID] = uuid # do we want the item's uuid in the rev's metadata?
         if CONTENTTYPE not in self:
             self[CONTENTTYPE] = u'application/octet-stream'
-        metas = self
+
+        if app.cfg.log_remote_addr:
+            remote_addr = request.remote_addr
+            if remote_addr:
+                self[ADDRESS] = unicode(remote_addr)
+                hostname = wikiutil.get_hostname(remote_addr)
+                if hostname:
+                    self[HOSTNAME] = hostname
+        try:
+            if flaskg.user.valid:
+                self[USERID] = unicode(flaskg.user.id)
+        except:
+            # when loading xml via script, we have no flaskg.user
+            pass
+
+        self.seek(0) # for a new revision, file pointer points to EOF, rewind first
+        rev_content = convert_to_indexable(self, new_rev=True)
+
         logging.debug("item %r revno %d update index:" % (name, revno))
-        for k, v in metas.items():
+        for k, v in self.items():
             logging.debug(" * rev meta %r: %r" % (k, v))
-        self._index.add_rev(uuid, revno, metas)
+        logging.debug("Indexable content: %r" % (rev_content[:250], ))
+        self._index.add_rev(uuid, revno, self, rev_content)
 
     def remove_index(self):
         """
@@ -195,6 +256,8 @@
     # TODO by intercepting write() to index data written to a revision
 
 from whoosh.writing import AsyncWriter
+from whoosh.qparser import QueryParser, MultifieldParser
+
 from MoinMoin.search.indexing import WhooshIndex
 
 class ItemIndex(object):
@@ -218,13 +281,12 @@
         """
         # XXX we do not have an index for item metadata yet!
 
-    def remove_item(self, metas):
+    def remove_item(self, uuid):
         """
         remove all data related to this item and all its revisions from the index
         """
         with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            doc_number = latest_revs_searcher.document_number(uuid=metas[UUID],
-                                                              name_exact=metas[NAME],
+            doc_number = latest_revs_searcher.document_number(uuid=uuid,
                                                               wikiname=self.wikiname
                                                              )
         if doc_number is not None:
@@ -232,8 +294,7 @@
                 async_writer.delete_document(doc_number)
 
         with self.index_object.all_revisions_index.searcher() as all_revs_searcher:
-            doc_numbers = list(all_revs_searcher.document_numbers(uuid=metas[UUID],
-                                                                  name_exact=metas[NAME],
+            doc_numbers = list(all_revs_searcher.document_numbers(uuid=uuid,
                                                                   wikiname=self.wikiname
                                                                  ))
         if doc_numbers:
@@ -241,7 +302,7 @@
                 for doc_number in doc_numbers:
                     async_writer.delete_document(doc_number)
 
-    def add_rev(self, uuid, revno, rev):
+    def add_rev(self, uuid, revno, rev, rev_content):
         """
         add a new revision <revno> for item <uuid> with metadata <metas>
         """
@@ -254,10 +315,6 @@
             latest_found_document = latest_revs_searcher.document(uuid=rev[UUID],
                                                                   wikiname=self.wikiname
                                                                  )
-        logging.debug("Processing: name %s revno %s" % (rev[NAME], revno))
-        rev.seek(0) # for a new revision, file pointer points to EOF, rewind first
-        rev_content = convert_to_indexable(rev)
-        logging.debug("Indexable content: %r" % (rev_content[:250], ))
         if not all_found_document:
             schema = self.index_object.all_revisions_index.schema
             with AsyncWriter(self.index_object.all_revisions_index) as async_writer:
@@ -280,6 +337,11 @@
                                                                      rev_no=revno,
                                                                      wikiname=self.wikiname
                                                                     )
+        if latest_doc_number is not None:
+            with AsyncWriter(self.index_object.latest_revisions_index) as async_writer:
+                logging.debug("Latest revisions: removing %d", latest_doc_number)
+                async_writer.delete_document(latest_doc_number)
+
         with self.index_object.all_revisions_index.searcher() as all_revs_searcher:
             doc_number = all_revs_searcher.document_number(uuid=uuid,
                                                            rev_no=revno,
@@ -289,40 +351,58 @@
             with AsyncWriter(self.index_object.all_revisions_index) as async_writer:
                 logging.debug("All revisions: removing %d", doc_number)
                 async_writer.delete_document(doc_number)
-        if latest_doc_number is not None:
-            with AsyncWriter(self.index_object.latest_revisions_index) as async_writer:
-                logging.debug("Latest revisions: removing %d", latest_doc_number)
-                async_writer.delete_document(latest_doc_number)
 
-    def history(self, mountpoint=u'', item_name=u'', reverse=True, start=None, end=None):
-        if mountpoint:
-            mountpoint += '/'
-        with self.index_object.all_revisions_index.searcher() as all_revs_searcher:
-            if item_name:
-                docs = all_revs_searcher.documents(name_exact=item_name,
-                                                   wikiname=self.wikiname
-                                                  )
-            else:
-                docs = all_revs_searcher.documents(wikiname=self.wikiname)
-            from operator import itemgetter
-            # sort by mtime and rev_no do deal better with mtime granularity for fast item rev updates
-            for doc in sorted(docs, key=itemgetter("mtime", "rev_no"), reverse=reverse)[start:end]:
-                yield (doc[MTIME], mountpoint + doc[NAME], doc["rev_no"])
+    def query_parser(self, default_fields, all_revs=False):
+        if all_revs:
+            schema = self.index_object.all_revisions_schema
+        else:
+            schema = self.index_object.latest_revisions_schema
+        if len(default_fields) > 1:
+            qp = MultifieldParser(default_fields, schema=schema)
+        elif len(default_fields) == 1:
+            qp = QueryParser(default_fields[0], schema=schema)
+        else:
+            raise ValueError("default_fields list must at least contain one field name")
+        return qp
 
-    def all_tags(self):
-        with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            docs = latest_revs_searcher.documents(wikiname=self.wikiname)
-            tags_names = {}
-            for doc in docs:
-                tags = doc.get(TAGS, [])
-                logging.debug("name %s rev %s tags %s" % (doc[NAME], doc["rev_no"], tags))
-                for tag in tags:
-                    tags_names.setdefault(tag, []).append(doc[NAME])
-            counts_tags_names = [(len(names), tag, names) for tag, names in tags_names.items()]
-            return counts_tags_names
+    def searcher(self, all_revs=False):
+        """
+        Get a searcher for the right index. Always use this with "with":
 
-    def tagged_items(self, tag):
-        with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            docs = latest_revs_searcher.documents(tags=tag, wikiname=self.wikiname)
-            return [doc[NAME] for doc in docs]
+        with storage.searcher(all_revs) as searcher:
+            # your code
 
+        If you do not need the searcher itself or the Result object, but rather
+        the found documents, better use search() or search_page(), see below.
+        """
+        if all_revs:
+            ix = self.index_object.all_revisions_index
+        else:
+            ix = self.index_object.latest_revisions_index
+        return ix.searcher()
+
+    def search(self, q, all_revs=False, **kw):
+        with self.searcher(all_revs) as searcher:
+            # Note: callers must consume everything we yield, so the for loop
+            # ends and the "with" is left to close the index files.
+            for hit in searcher.search(q, **kw):
+                yield hit.fields()
+
+    def search_page(self, q, all_revs=False, pagenum=1, pagelen=10, **kw):
+        with self.searcher(all_revs) as searcher:
+            # Note: callers must consume everything we yield, so the for loop
+            # ends and the "with" is left to close the index files.
+            for hit in searcher.search_page(q, pagenum, pagelen=pagelen, **kw):
+                yield hit.fields()
+
+    def documents(self, all_revs=False, **kw):
+        if all_revs:
+            ix = self.index_object.all_revisions_index
+        else:
+            ix = self.index_object.latest_revisions_index
+        with ix.searcher() as searcher:
+            # Note: callers must consume everything we yield, so the for loop
+            # ends and the "with" is left to close the index files.
+            for doc in searcher.documents(**kw):
+                yield doc
+
--- a/MoinMoin/storage/backends/sqla.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/backends/sqla.py	Thu Sep 01 00:16:33 2011 +0200
@@ -49,9 +49,7 @@
 
         * Data.read must be changed to operate on dynamically loaded chunks. I.e., the data._chunks must
           be set to lazy='dynamic', which will then be a query instead of a collection.
-        * Find a proper solution for methods that issue many SQL queries. Especially search_items is
-          difficult, as we cannot know what data will be needed in the subsequent processing of the items
-          returned, which will result in more queries being issued. Eager loading is only a partial solution.
+        * Find a proper solution for methods that issue many SQL queries.
         * MetaData should definitely NOT simply be stored as a dict in a PickleType Column. Store that properly,
           perhaps in (a) seperate table(s).
         * Find out why RC lists an item that was just written below Trash/ as well. (Likely a UI bug.)
--- a/MoinMoin/storage/serialization.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/storage/serialization.py	Thu Sep 01 00:16:33 2011 +0200
@@ -155,26 +155,13 @@
         return item is not None and item.name in self.item_names
 
 
-class TermMatch(XMLSelectiveGenerator):
-    def __init__(self, out, term):
-        self.term = term  # see MoinMoin.storage.terms
-        XMLSelectiveGenerator.__init__(self, out)
-
-    def shall_serialize(self, item=None, rev=None,
-                        revno=None, current_revno=None):
-        if item is not None:
-            self.term.prepare()
-            return self.term.evaluate(item)
-        return False
-
-
 def serialize(obj, xmlfile, xmlgen_cls=XMLSelectiveGenerator, *args, **kwargs):
     """
     Serialize <obj> to <xmlfile>.
 
     The default value of <xmlgen_cls> will just serialize everything. Alternatively,
     use some of XMLSelectiveGenerator child classes to do selective serialization,
-    e.g. of just a list of items or just of items that match some search term.
+    e.g. of just a list of items or just a subset of the revisions.
 
     :arg obj: object to serialize (must mix in Serializable)
     :arg xmlfile: output file (file-like or filename)
--- a/MoinMoin/storage/terms.py	Wed Aug 24 11:29:29 2011 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,459 +0,0 @@
-# Copyright: 2008 MoinMoin:JohannesBerg
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-    MoinMoin - search expression object representation
-
-    This module defines the possible search terms for a query to the
-    storage backend. This is used, for example, to implement searching,
-    page lists etc.
-
-    Note that some backends can optimise some of the search terms, for
-    example a backend that has indexed various metadata keys can optimise
-    easy expressions containing ItemMetaDataMatch terms. This is only allowed
-    for classes documented as being 'final' which hence also means that
-    their _evaluate function may not be overridden by descendent classes.
-
-    For example, that metadata backend could test if the expression is an
-    ItemMetaDataMatch expression, and if so, simply return the appropriate
-    index; or if it is an AND() expression build the page list from the
-    index, remove the ItemMetaDataMatch instance from the AND list and match
-    the resulting expression only for pages in that list. Etc.
-
-    TODO: Should we write some generic code for picking apart expressions
-          like that?
-"""
-
-
-import re
-
-from MoinMoin.storage.error import NoSuchRevisionError
-
-# Base classes
-
-class Term(object):
-    """
-    Base class for search terms.
-    """
-    # relative cost of this search term
-    _cost = 0
-
-    def __init__(self):
-        pass
-
-    def evaluate(self, item):
-        """
-        Evaluate this term and return True or False if the
-        item identified by the parameters matches.
-
-        :param item: the item
-        """
-        assert hasattr(self, '_result')
-
-        if self._result is None:
-            self._result = self._evaluate(item)
-
-        return self._result
-
-    def _evaluate(self, item):
-        """
-        Implements the actual evaluation
-        """
-        raise NotImplementedError()
-
-    def prepare(self):
-        """
-        Prepare this search term to make it ready for testing.
-        Must be called before each outermost-level evaluate.
-        """
-        self._result = None
-
-    def copy(self):
-        """
-        Make a copy of this search term.
-        """
-        return self.__class__()
-
-class UnaryTerm(Term):
-    """
-    Base class for search terms that has a single contained
-    search term, e.g. NOT.
-    """
-    def __init__(self, term):
-        Term.__init__(self)
-        assert isinstance(term, Term)
-        self.term = term
-
-    def prepare(self):
-        Term.prepare(self)
-        self.term.prepare()
-        self._cost = self.term._cost
-
-    def __repr__(self):
-        return u'<%s(%r)>' % (self.__class__.__name__, self.term)
-
-    def copy(self):
-        return self.__class__(self.term.copy())
-
-class ListTerm(Term):
-    """
-    Base class for search terms that contain multiple other
-    search terms, e.g. AND.
-    """
-    def __init__(self, *terms):
-        Term.__init__(self)
-        for e in terms:
-            assert isinstance(e, Term)
-        self.terms = list(terms)
-
-    def prepare(self):
-        Term.prepare(self)
-        # the sum of all costs is a bit of a worst-case cost...
-        self._cost = 0
-        for e in self.terms:
-            e.prepare()
-            self._cost += e._cost
-        self.terms.sort(cmp=lambda x, y: cmp(x._cost, y._cost))
-
-    def remove(self, subterm):
-        self.terms.remove(subterm)
-
-    def add(self, subterm):
-        self.terms.append(subterm)
-
-    def __repr__(self):
-        return u'<%s(%s)>' % (self.__class__.__name__,
-                              ', '.join([repr(t) for t in self.terms]))
-
-    def copy(self):
-        terms = [t.copy() for t in self.terms]
-        return self.__class__(*terms)
-
-# Logical expression classes
-
-class AND(ListTerm):
-    """
-    AND connection between multiple terms. Final.
-    """
-    def _evaluate(self, item):
-        for e in self.terms:
-            if not e.evaluate(item):
-                return False
-        return True
-
-class OR(ListTerm):
-    """
-    OR connection between multiple terms. Final.
-    """
-    def _evaluate(self, item):
-        for e in self.terms:
-            if e.evaluate(item):
-                return True
-        return False
-
-class NOT(UnaryTerm):
-    """
-    Inversion of a single term. Final.
-    """
-    def _evaluate(self, item):
-        return not self.term.evaluate(item)
-
-class XOR(ListTerm):
-    """
-    XOR connection between multiple terms, i.e. exactly
-    one must be True. Final.
-    """
-    def _evaluate(self, item):
-        count = 0
-        for e in self.terms:
-            if e.evaluate(item):
-                count += 1
-        return count == 1
-
-class _BOOL(Term):
-    _cost = 0
-    def __init__(self, val):
-        self._val = val
-
-    def prepare(self):
-        self._result = self._val
-
-    def __repr__(self):
-        return '<%s>' % str(self._val).upper()
-
-    def copy(self):
-        return self
-
-TRUE = _BOOL(True)
-FALSE = _BOOL(False)
-
-def BOOL(b):
-    if b:
-        return TRUE
-    return FALSE
-
-# Actual Moin search terms
-
-class TextRE(Term):
-    """
-    Regular expression full text match, use as last resort.
-    """
-    _cost = 1000 # almost prohibitive
-    def __init__(self, needle_re):
-        Term.__init__(self)
-        assert hasattr(needle_re, 'search')
-        self._needle_re = needle_re
-
-    def _evaluate(self, item):
-        try:
-            rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            return False
-        data = rev.read()
-        return not (not self._needle_re.search(data))
-
-    def __repr__(self):
-        return u'<term.TextRE(...)>'
-
-    def copy(self):
-        return TextRE(self._needle_re)
-
-class Text(TextRE):
-    """
-    Full text match including middle of words and over word
-    boundaries. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile(re.escape(needle), flags)
-        TextRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.Text(%s, %s)>' % (self.needle, self.case_sensitive)
-
-    def copy(self):
-        return Text(self.needle, self.case_sensitive)
-
-class Word(TextRE):
-    """
-    Full text match finding exact words. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile('\\b' + re.escape(needle) + '\\b', flags)
-        TextRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.Word(%s, %s)>' % (self.needle, self.case_sensitive)
-
-    def copy(self):
-        return Word(self.needle, self.case_sensitive)
-
-class WordStart(TextRE):
-    """
-    Full text match finding the start of a word. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile('\\b' + re.escape(needle), flags)
-        TextRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.WordStart(%s, %s)>' % (self.needle, self.case_sensitive)
-
-    def copy(self):
-        return WordStart(self.needle, self.case_sensitive)
-
-class WordEnd(TextRE):
-    """
-    Full text match finding the end of a word. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile(re.escape(needle) + '\\b', flags)
-        TextRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.WordEnd(%s, %s)>' % (self.needle, self.case_sensitive)
-
-    def copy(self):
-        return WordEnd(self.needle, self.case_sensitive)
-
-class NameRE(Term):
-    """
-    Matches the item's name with a given regular expression.
-    """
-    _cost = 10 # one of the cheapest
-    def __init__(self, needle_re):
-        Term.__init__(self)
-        assert hasattr(needle_re, 'search')
-        self._needle_re = needle_re
-
-    def _evaluate(self, item):
-        return not (not self._needle_re.search(item.name))
-
-    def __repr__(self):
-        return u'<term.NameRE(...)>'
-
-    def copy(self):
-        return NameRE(self._needle_re)
-
-class Name(NameRE):
-    """
-    Item name match, given needle must occur in item's name. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        assert isinstance(needle, unicode)
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile(re.escape(needle), flags)
-        NameRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.Name(%s, %s)>' % (self.needle, self.case_sensitive)
-
-    def copy(self):
-        return Name(self.needle, self.case_sensitive)
-
-class NameFn(Term):
-    """
-    Arbitrary item name matching function.
-    """
-    def __init__(self, fn):
-        Term.__init__(self)
-        assert callable(fn)
-        self._fn = fn
-
-    def _evaluate(self, item):
-        return not (not self._fn(item.name))
-
-    def __repr__(self):
-        return u'<term.NameFn(%r)>' % (self._fn, )
-
-    def copy(self):
-        return NameFn(self._fn)
-
-class ItemMetaDataMatch(Term):
-    """
-    Matches a metadata key/value pair of an item, requires
-    existence of the metadata key. Final.
-    """
-    _cost = 100 # fairly expensive but way cheaper than text
-    def __init__(self, key, val):
-        Term.__init__(self)
-        self.key = key
-        self.val = val
-
-    def _evaluate(self, item):
-        return self.key in item and item[self.key] == self.val
-
-    def __repr__(self):
-        return u'<%s(%s: %s)>' % (self.__class__.__name__, self.key, self.val)
-
-    def copy(self):
-        return ItemMetaDataMatch(self.key, self.val)
-
-class ItemHasMetaDataValue(Term):
-    """
-    Match when the metadata value for a given key contains the given
-    value (when the item's metadata value is a dict or list), requires
-    existence of the metadata key. Final.
-    """
-    _cost = 100 # fairly expensive but way cheaper than text
-    def __init__(self, key, val):
-        Term.__init__(self)
-        self.key = key
-        self.val = val
-
-    def _evaluate(self, item):
-        return self.key in item and self.val in item[self.key]
-
-    def __repr__(self):
-        return u'<%s(%s: %s)>' % (self.__class__.__name__, self.key, self.val)
-
-    def copy(self):
-        return ItemHasMetaDataValue(self.key, self.val)
-
-class ItemHasMetaDataKey(Term):
-    """
-    Requires existence of the metadata key. Final.
-    """
-    _cost = 90 # possibly cheaper than ItemMetaDataMatch
-    def __init__(self, key):
-        Term.__init__(self)
-        self.key = key
-
-    def _evaluate(self, item):
-        return self.key in item
-
-    def __repr__(self):
-        return u'<%s(%s)>' % (self.__class__.__name__, self.key)
-
-    def copy(self):
-        return ItemHasMetaDataKey(self.key)
-
-class LastRevisionMetaDataMatch(Term):
-    """
-    Matches a metadata key/value pair of an item, requires
-    existence of the metadata key. Final.
-    """
-    _cost = 100 # fairly expensive but way cheaper than text
-    def __init__(self, key, val):
-        Term.__init__(self)
-        self.key = key
-        self.val = val
-
-    def _evaluate(self, item):
-        try:
-            rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            return False
-        return self.key in rev and rev[self.key] == self.val
-
-    def __repr__(self):
-        return u'<%s(%s: %s)>' % (self.__class__.__name__, self.key, self.val)
-
-    def copy(self):
-        return LastRevisionMetaDataMatch(self.key, self.val)
-
-class LastRevisionHasMetaDataKey(Term):
-    """
-    Requires existence of the metadata key. Final.
-    """
-    _cost = 90 # possibly cheaper than LastRevisionMetaDataMatch
-    def __init__(self, key):
-        Term.__init__(self)
-        self.key = key
-
-    def _evaluate(self, item):
-        try:
-            rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            return False
-        return self.key in rev
-
-    def __repr__(self):
-        return u'<%s(%s)>' % (self.__class__.__name__, self.key)
-
-    def copy(self):
-        return LastRevisionHasMetaDataKey(self.key)
-
--- a/MoinMoin/templates/global_history.html	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/templates/global_history.html	Thu Sep 01 00:16:33 2011 +0200
@@ -17,16 +17,6 @@
 </div>
 <h1>{{ _("Global History") }}</h1>
 <div class='moin-clr'></div>
-    {% if user.valid  %}
-    <div id="moin-set-bookmark">
-        {% if bookmark_time %}
-            {{ _("Bookmark (currently set to %(bookmark)s). ", bookmark=bookmark_time|datetimeformat) }} 
-            <a href="{{ url_for('frontend.bookmark', time='del') }}">{{ _("Delete bookmark") }}</a>
-        {% else %}
-            {{ _("Bookmark (not set). ") }}<a href="{{ url_for('frontend.bookmark', time=current_timestamp) }}">{{ _("Set bookmark") }}</a> 
-        {% endif %}
-    </div>
-    {% endif %}
     <div id="moin-global-history">
         {% for rev_date, revs in history %}
            {% set  latest_rev = revs[0] %}
@@ -36,7 +26,7 @@
                     <span>
                         <h2>{{ rev_date }}</h2>
                         {% if user.valid %}
-                        <a class="bookmark-link" href="{{ url_for('frontend.bookmark', time=latest_timestamp) }}">{{ _("Set bookmark") }}</a>
+                        <a class="bookmark-link" href="{{ url_for('frontend.bookmark', time=utctimestamp(latest_timestamp)) }}">{{ _("Set bookmark") }}</a>
                         {% endif %}
                    </span>
                 </div>
@@ -78,8 +68,15 @@
                 </div>
             </div>
         {% endfor %}
+        {% if user.valid and bookmark_time %}
+        <div class="moin-history-container"> 
+            <div class="moin-history-container-header">
+                <span>
+                    <h2>{{ bookmark_time|datetimeformat }}</h2>
+                    <a href="{{ url_for('frontend.bookmark', time='del') }}">{{ _("Delete bookmark") }}</a>
+               </span>
+            </div>
+        </div>
+        {% endif %}
     </div>
-   {% if bookmark_time and not offset %}
-       <div id="moin-bookmark-reached">{{ _("Bookmark reached") }}</div>
-   {% endif %}
 {% endblock %}
--- a/MoinMoin/templates/history.html	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/templates/history.html	Thu Sep 01 00:16:33 2011 +0200
@@ -27,27 +27,27 @@
                 <th>{{ _("Comment") }}</th>
                 <th colspan="6">{{ _("Actions") }}</th>
             </tr>
-            {% for rev in history %}
+            {% for doc in history %}
             <tr>
-                <td class="moin-wordbreak">{{ rev.name }}</td>
-                <td class="moin-integer">{{ rev.revno }}</td>
-                <td>{{ rev.timestamp|datetimeformat }}</td>
-                <td class="moin-integer">{{ rev.size }}</td>
+                <td class="moin-wordbreak">{{ doc.name }}</td>
+                <td class="moin-integer">{{ doc.rev_no }}</td>
+                <td>{{ doc.mtime|datetimeformat }}</td>
+                <td class="moin-integer">{{ doc.size }}</td>
                 <td>
                     <div class="moin-hist-rev">
-                        <input type="radio" name="rev1" value="{{ rev.revno }}" />
-                        <input type="radio" name="rev2" value="{{ rev.revno }}" />
+                        <input type="radio" name="rev1" value="{{ doc.rev_no }}" />
+                        <input type="radio" name="rev2" value="{{ doc.rev_no }}" />
                     </div>
                 </td>
-                <td class="moin-wordbreak">{{ utils.editor_info(rev) }}</td>
-                <td class="moin-wordbreak">{{ rev.contenttype }}</td>
-                <td class="moin-wordbreak">{{ rev.comment }}</td>
-                <td><a href="{{ url_for('frontend.show_item', item_name=rev.item.name, rev=rev.revno) }}">{{ _('show') }}</a></td>
-                <td><a href="{{ url_for('frontend.show_item_meta', item_name=rev.item.name, rev=rev.revno) }}">{{ _('meta') }}</a></td>
-                <td><a href="{{ url_for('frontend.download_item', item_name=rev.item.name, rev=rev.revno) }}">{{ _('download') }}</a></td>
-                <td><a href="{{ url_for('frontend.highlight_item', item_name=rev.item.name, rev=rev.revno) }}">{{ _('highlight') }}</a></td>
-                <td><a href="{{ url_for('frontend.revert_item', item_name=rev.item.name, rev=rev.revno) }}">{{ _('revert') }}</a></td>
-                <td><a href="{{ url_for('frontend.destroy_item', item_name=rev.item.name, rev=rev.revno) }}">{{ _('destroy') }}</a></td>
+                <td class="moin-wordbreak">{{ utils.editor_info(doc) }}</td>
+                <td class="moin-wordbreak">{{ doc.contenttype }}</td>
+                <td class="moin-wordbreak">{{ doc.comment }}</td>
+                <td><a href="{{ url_for('frontend.show_item', item_name=doc.name, rev=doc.rev_no) }}">{{ _('show') }}</a></td>
+                <td><a href="{{ url_for('frontend.show_item_meta', item_name=doc.name, rev=doc.rev_no) }}">{{ _('meta') }}</a></td>
+                <td><a href="{{ url_for('frontend.download_item', item_name=doc.name, rev=doc.rev_no) }}">{{ _('download') }}</a></td>
+                <td><a href="{{ url_for('frontend.highlight_item', item_name=doc.name, rev=doc.rev_no) }}">{{ _('highlight') }}</a></td>
+                <td><a href="{{ url_for('frontend.revert_item', item_name=doc.name, rev=doc.rev_no) }}">{{ _('revert') }}</a></td>
+                <td><a href="{{ url_for('frontend.destroy_item', item_name=doc.name, rev=doc.rev_no) }}">{{ _('destroy') }}</a></td>
             </tr>
             {% endfor %}
         </table>
--- a/MoinMoin/templates/layout.html	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/templates/layout.html	Thu Sep 01 00:16:33 2011 +0200
@@ -19,11 +19,10 @@
 <div id="moin-header">
 {% block header %}
     {% if search_form %} 
-    {{ gen.form.open(search_form, id='moin-searchform', method='get', action=url_for('frontend.show_item', item_name=item_name)) }}
+    {{ gen.form.open(search_form, id='moin-searchform', method='get', action=url_for('frontend.search')) }}
         <div>
-            {{ gen.input(search_form['q'], type='search', id='moin-search-query', size='30') }}
-            {{ gen.input(search_form['submit'], type='submit') }}
-            {{ gen.input(search_form['pagelen'], type='hidden', value='25') }}
+            {{ gen.input(search_form['q'], type='search', id='moin-search-query', size='40') }}
+            {{ gen.input(search_form['submit'], type='submit', id='moin-search-submit') }}
             {{ forms.render_errors(search_form) }}
         </div>
     {{ gen.form.close() }}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/templates/search.html	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,63 @@
+{% extends theme("layout.html") %}
+{% import "utils.html" as utils %}
+{% block content %}
+    {% if results is defined %}
+    <p class="searchstats">
+        {% if results %}
+        {{ _("%(result_len)d results found (%(runtime).3f secs).",
+              result_len=results|length, runtime=results.runtime
+            )
+        }}
+        {% else %}
+        {{ _("No results found (%(runtime).3f secs).", runtime=results.runtime) }}
+        {% endif %}
+    </p>
+    {% endif %}
+    {{ gen.form.open(medium_search_form, id='moin-long-searchform', method='get', action=url_for('frontend.search')) }}
+        <div>
+            {{ gen.input(medium_search_form['q'], type='search', id='moin-search-query') }}
+            {{ gen.input(medium_search_form['submit'], type='submit') }}
+            {{ forms.render_field_without_markup(gen, medium_search_form['history'], 'checkbox') }}
+            {{ forms.render_errors(medium_search_form) }}
+        </div>
+    {{ gen.form.close() }}
+    {% if results is defined %}
+    <div>
+        <p>{{ _("name term suggestions: %(termlist)s", termlist=name_suggestions) }}</p>
+        <p>{{ _("content term suggestions: %(termlist)s", termlist=content_suggestions) }}</p>
+    </div>
+    <div class="searchresults">
+        <table>
+            {% for result in results %}
+                {% if result['wikiname'] == cfg.interwikiname %}
+                    <tr>
+                        <td class="moin-wordbreak">{{ result.pos + 1 }}
+                        <a href="{{ url_for_item(item_name=result['name'], wiki_name='Self', rev=result['rev_no']) }}"><b>{{ result['name'] }}</b></a>
+                        </td>
+                    </tr>
+                    <tr>
+                        <td>
+                            <p class="info searchhitinfobar">{{ _("Revision: %(rev_no)d Last Change: %(mtime)s", rev_no=result['rev_no'], mtime=result['mtime']|datetimeformat) }}</p>
+                        </td>
+                    </tr>
+                    <tr>
+                        <td>
+                            {% if user.may.read(result['name']) %}
+                                <p class="info foundtext">{{ result.highlights('content')|safe }}</p>
+                            {% else %}
+                                <p class="info foundtext">{{ _("You don't have read permission for this item.") }}</p>
+                            {% endif %}
+                        </td>
+                    </tr>
+                {% else %}
+                    <tr>
+                        <td class="moin-wordbreak">{{ result.pos + 1 }}
+                        <a class="moin-interwiki" href="{{ url_for_item(item_name=result['name'], wiki_name=result['wikiname'], rev=result['rev_no']) }}"><b>{{ "%s:%s" % (result['wikiname'], result['name']) }}</b></a>
+                        </td>
+                    </tr>
+                {% endif %}
+            {% endfor %}
+        </table>
+    </div>
+    {% endif %}
+{% endblock %}
--- a/MoinMoin/templates/search_results.html	Wed Aug 24 11:29:29 2011 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,61 +0,0 @@
-{% extends theme("layout.html") %}
-{% import "utils.html" as utils %}
-{% block content %}
-    {% if results %}
-    <p class="searchstats">
-        {{ _("Result: Page %(start_page)d of %(end_page)d.
-              Showing results %(start_result)d - %(end_result)d of %(result_len)d (%(runtime).3f secs)",
-              start_page=results.pagenum, end_page=results.pagecount,
-              start_result=results.offset + 1, end_result=results.offset + results.pagelen,
-              result_len=results|length, runtime=results.results.runtime
-            )
-        }}
-    {% endif %}
-    {{ gen.form.open(medium_search_form, id='moin-long-searchform', method='get', action=url_for('frontend.show_item', item_name=item_name)) }}
-        <div>
-            {{ gen.input(medium_search_form['q'], type='search', id='moin-search-query') }}
-            {{ gen.input(medium_search_form['submit'], type='submit') }}
-            {{ gen.input(medium_search_form['pagelen'], type='hidden', value='25') }}
-            {{ forms.render_field_without_markup(gen, medium_search_form['search_in_all'], 'checkbox') }}
-            {{ forms.render_errors(medium_search_form) }}
-        </div>
-    {{ gen.form.close() }}
-    </p>
-        {% if results %}
-        <div class="searchresults">
-        <table>
-            {% for result in results %}
-                {% if result['wikiname'] == cfg.interwikiname %}
-                    <tr>
-                        <td class="moin-wordbreak">{{ result.pos + 1 }}
-                        <a href="{{ url_for_item(item_name=result['name'], wiki_name='Self', rev=result['rev_no']) }}"><b>{{ result['name'] }}</b></a>
-                        </td>
-                    </tr>
-                    <tr>
-                        <td>
-                            <p class="info searchhitinfobar">{{ _("Revision: %(rev_no)d Last Change: %(mtime)s", rev_no=result['rev_no'], mtime=result['mtime']|datetimeformat) }}</p>
-                        </td>
-                    </tr>
-                    <tr>
-                        <td>
-                            {% if user.may.read(result['name']) %}
-                                <p class="info foundtext">{{ result.highlights('content')|safe }}</p>
-                            {% else %}
-                                <p class="info foundtext">{{ _("You don't have read permission for this item.") }}</p>
-                            {% endif %}
-                        </td>
-                    </tr>
-                {% else %}
-                    <tr>
-                        <td class="moin-wordbreak">{{ result.pos + 1 }}
-                        <a class="moin-interwiki" href="{{ url_for_item(item_name=result['name'], wiki_name=result['wikiname'], rev=result['rev_no']) }}"><b>{{ "%s:%s" % (result['wikiname'], result['name']) }}</b></a>
-                        </td>
-                    </tr>
-                {% endif %}
-            {% endfor %}
-        </table>
-        </div>
-    {% else %}
-        <h1>{{ _("No results for '%(query)s'", query=query) }}</h1>
-    {% endif %}
-{% endblock %}
--- a/MoinMoin/templates/wanteds.html	Wed Aug 24 11:29:29 2011 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,20 +0,0 @@
-{% extends theme("layout.html") %}
-{% block content %}
-    {% if headline %}
-        <h1>{{ headline }}</h1>
-    {% endif %}
-    {% if wanteds %}
-        {{ _("Total: %(total)s", total=wanteds|count) }}
-        <ol type="1">
-            {% for item_name in wanteds|sort %}
-                <li>
-                    <a href="{{ url_for('frontend.show_item', item_name=item_name) }}">{{ item_name }}</a>:
-                    {% for item_link in wanteds[item_name] %}
-                        <a href="{{ url_for('frontend.show_item', item_name=item_link) }}">{{ item_link }}</a>
-                        {%- if not loop.last %}, {% endif -%}
-                    {% endfor %}
-                </li>
-            {% endfor %}
-        </ol>
-    {% endif %}
-{% endblock %}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/test_logging.conf	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,24 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=stderr
+
+[formatters]
+keys=tests
+
+[logger_root]
+level=DEBUG
+handlers=stderr
+
+[handler_stderr]
+class=StreamHandler
+level=ERROR
+formatter=tests
+args=(sys.stderr, )
+
+[formatter_tests]
+format=%(asctime)s %(levelname)s %(name)s:%(lineno)d %(message)s
+datefmt=
+class=logging.Formatter
+
--- a/MoinMoin/themes/__init__.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/themes/__init__.py	Thu Sep 01 00:16:33 2011 +0200
@@ -366,6 +366,17 @@
     return 'moin-mime-%s' % cls
 
 
+def utctimestamp(dt):
+    """
+    convert a datetime object (UTC) to a UNIX timestamp (UTC)
+
+    Note: time library writers seem to have a distorted relationship to inverse
+          functions and also to UTC (see time.gmtime, see datetime.utcfromtimestamp).
+    """
+    from calendar import timegm
+    return timegm(dt.timetuple())
+
+
 def setup_jinja_env():
     app.jinja_env.filters['shorten_item_name'] = shorten_item_name
     app.jinja_env.filters['contenttype_to_class'] = contenttype_to_class
@@ -387,6 +398,7 @@
                             'item_name': 'handlers need to give it',
                             'url_for_item': url_for_item,
                             'get_editor_info': lambda rev: get_editor_info(rev),
+                            'utctimestamp': lambda dt: utctimestamp(dt),
                             'gen': make_generator(),
                             })
 
--- a/MoinMoin/themes/_tests/test_navi_bar.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/themes/_tests/test_navi_bar.py	Thu Sep 01 00:16:33 2011 +0200
@@ -11,7 +11,7 @@
 
 from MoinMoin._tests import wikiconfig
 from MoinMoin.themes import ThemeSupport
-
+from MoinMoin import themes
 
 class TestNaviBar(object):
     class Config(wikiconfig.Config):
@@ -35,3 +35,34 @@
             result = self.theme.split_navilink(navilink)
             assert result == expected
 
+    def test_location_breadcrumbs(self):
+        test_result = ThemeSupport.location_breadcrumbs(self.theme, 'some/place/test_item')
+        test_segment_name_1, test_item_name_1, test_item_exists_1 = test_result[0]
+        test_segment_name_2, test_item_name_2, test_item_exists_2 = test_result[1]
+        test_segment_name_3, test_item_name_3, test_item_exists_3 = test_result[2]
+
+        assert test_segment_name_1 == 'some'
+        assert test_item_name_1 == 'some'
+        assert test_segment_name_2 == 'place'
+        assert test_item_name_2 == 'some/place'
+        assert test_segment_name_3 == 'test_item'
+        assert test_item_name_3 == 'some/place/test_item'
+
+    def test_parent_item(self):
+        test_result = ThemeSupport.parent_item(self.theme, 'moin/moin-2.0/Item')
+        expected = 'moin/moin-2.0'
+        assert test_result == expected, ('Expected "%(expected)s" but got "%(test_result)s"') % locals()
+
+def test_shorten_item_name():
+    test_result1 = themes.shorten_item_name(u'MoinMoin/some/value', 0)
+    assert test_result1 == u'valu...lue'
+    test_result2 = themes.shorten_item_name(u'MoinMoin/some/value', 5)
+    assert test_result2 == 'value'
+    test_result3 = themes.shorten_item_name(u'MoinMoin/some/value')
+    assert test_result3 == u'MoinMoin/some/value'
+
+def test_contenttype_to_class():
+    test_result = themes.contenttype_to_class(u'MoinMoin/some/value')
+    expected = u'moin-mime-MoinMoin'
+    assert test_result == expected
+
--- a/MoinMoin/themes/modernized/static/css/common.css	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/themes/modernized/static/css/common.css	Thu Sep 01 00:16:33 2011 +0200
@@ -571,10 +571,11 @@
             line-height: 1.12em; }
 
 /* moin-header searchform */
-#moin-searchform { margin: 8px .5em; padding: 0; font-size: 0.82em; float: right;  text-align: left; }
+#moin-searchform { margin: 8px .5em; padding: 0; font-size: 0.82em; float: right;  text-align: right; }
 #moin-searchform input { font-size: 100%; vertical-align: middle;
             background-color: #F3F7FD; /* same as body bg col */
             border: 1px solid #A4B9DF; }
+#moin-searchform #moin-search-submit { display: none; }
 #moin-searchform div { margin: 0; }
 #moin-searchform ul { border: 1px solid #A4B9DF; margin: 0; padding: 0; background-color: #F3F7FD; }
 #moin-searchform li { list-style:none; }
--- a/MoinMoin/user.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/user.py	Thu Sep 01 00:16:33 2011 +0200
@@ -105,19 +105,21 @@
     return [item.name for item in all_users]
 
 
-def get_by_filter(key, value):
-    """ Searches for an user with a given filter """
-    from MoinMoin.storage.terms import ItemMetaDataMatch
-    items = get_user_backend().search_items(ItemMetaDataMatch(key, value))
-    users = [User(item.name) for item in items]
-    return users
+def get_items_by_filter(key, value):
+    """ Searches for a user with a given filter """
+    backend = get_user_backend()
+    items_found = []
+    for item in backend.iteritems():
+        if item.get(key) == value:
+            items_found.append(item)
+    return items_found
 
 
 def get_by_email_address(email_address):
     """ Searches for an user with a particular e-mail address and returns it. """
-    users = get_by_filter('email', email_address)
-    if len(users) > 0:
-        return users[0]
+    items = get_items_by_filter('email', email_address)
+    if items:
+        return User(items[0].name)
 
 def get_by_openid(openid):
     """
@@ -128,9 +130,9 @@
     :returns: the user whose openid is this one
     :rtype: user object or None
     """
-    users = get_by_filter('openid', openid)
-    if len(users) > 0:
-        return users[0]
+    items = get_items_by_filter('openid', openid)
+    if items:
+        return User(items[0].name)
 
 def getUserId(searchName):
     """ Get the user ID for a specific user NAME.
@@ -139,15 +141,9 @@
     :rtype: string
     :returns: the corresponding user ID or None
     """
-    from MoinMoin.storage.terms import ItemMetaDataMatch
-    try:
-        backend = get_user_backend()
-        for user in backend.search_items(ItemMetaDataMatch('name', searchName)):
-            return user.name
-        return None
-    except IndexError:
-        return None
-
+    items = get_items_by_filter('name', searchName)
+    if items:
+        return items[0].name
 
 def get_editor(userid, addr, hostname):
     """ Return a tuple of type id and string or Page object
--- a/MoinMoin/util/_tests/test_crypto.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_crypto.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,8 +7,7 @@
 """
 
 
-import py
-
+import pytest
 from MoinMoin.util import crypto
 
 
@@ -22,6 +21,9 @@
         result2 = crypto.random_string(length)
         assert result1 != result2, ('Expected different random strings, but got "%(result1)s" and "%(result2)s"') % locals()
 
+        result_string = crypto.random_string(length)
+        assert isinstance(result_string, str), ('Expected an string value, but got ' + str(type(result_string)))
+
         result = len(crypto.random_string(length))
         expected = length
         assert result == expected, ('Expected length "%(expected)s" but got "%(result)s"') % locals()
@@ -46,6 +48,51 @@
         expected = "{SSHA256}pdYvYv+4Vph259sv/HAm7zpZTv4sBKX9ITOX/m00HMsxMjM0NQ=="
         assert result == expected
 
+    def testupgradepassword(self):
+        """ return new password hash with better hash """
+        result = crypto.upgrade_password(u'MoinMoin', "junk_hash")
+        assert result.startswith('{SSHA256}')
+
+    def testvalidpassword(self):
+        """ validate user password """
+        hash_val = crypto.crypt_password(u"MoinMoin", salt='12345')
+        result = crypto.valid_password(u'MoinMoin', hash_val)
+        assert result
+        with pytest.raises(ValueError):
+            invalid_result = crypto.valid_password("MoinMoin", '{junk_value}')
+
+
+class TestToken(object):
+    """ tests for the generated tokens """
+
+    def test_validtoken(self):
+        """ validate the token """
+        test_key, test_token = crypto.generate_token(key='MoinMoin') # having some key value
+        result = crypto.valid_token(test_key, test_token)
+        assert result
+
+        test_key, test_token = crypto.generate_token() # key value is none
+        result = crypto.valid_token(test_key, test_token)
+        assert result
+
+        test_parts = test_token.split('-')
+        test_parts[0] = 'not_valid'
+        # changed value of the first part, should not be string
+        test_token_changed = '-'.join(test_parts)
+        result = crypto.valid_token(test_key, test_token_changed)
+        assert not result
+
+        test_key, test_token = 'MoinMoin', 'incorrect_token'
+        result = crypto.valid_token(test_key, test_token)
+        assert not result
+
+    def test_cache_key(self):
+        """ The key must be different for different <kw> """
+        test_kw1 = {'MoinMoin': 'value1'}
+        result1 = crypto.cache_key(**test_kw1)
+        test_kw2 = {'Moin2': 'value2'}
+        result2 = crypto.cache_key(**test_kw2)
+        assert result1 != result2, ("Expected different keys for different <kw> but got the same")
 
 coverage_modules = ['MoinMoin.util.crypto']
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_diff_html.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - MoinMoin.util.diff_html Tests
+"""
+
+import pytest
+from MoinMoin.util import diff_html
+
+def test_indent():
+    # input text
+    test_input = """ \n
+
+
+AAA 001
+AAA 002
+AAA 003
+AAA 004
+AAA 005
+"""
+    # expeted result
+    expected = """&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;AAA 001
+AAA 002
+AAA 003
+AAA 004
+AAA 005
+"""
+    result = diff_html.indent(test_input)
+    assert result == expected
+
+def test_diff():
+    test_input1 = """ \n
+
+
+AAA 001
+AAA 002
+AAA 003
+AAA 004
+AAA 005
+"""
+
+    # Case 1: charobj.ratio() < 0.5 i.e. Insufficient similarity
+    test_input2 = """ \n
+
+BBB 006
+BBB 007
+BBB 008
+BBB 009
+BBB 100
+"""
+    result = diff_html.diff(test_input1, test_input2)
+    expected = [(4, '<span><br>AAA 001<br>AAA 002<br>AAA 003<br>AAA 004<br>AAA 005</span>',
+                 4, '<span>BBB 006<br>BBB 007<br>BBB 008<br>BBB 009<br>BBB 100</span>')]
+    assert result == expected
+
+    #Case 2 : charobj.ratio() > 0.5 i.e. Some similarities
+    test_input3 = """ \n
+
+AAA 006
+AAA 007
+AAA 008
+AAA 009
+AAA 100
+"""
+    result = diff_html.diff(test_input1, test_input3)
+    expected = [(4, '<br>AAA 00<span>1</span><br>AAA 00<span>2</span><br>AAA 00<span>3</span><br>AAA 00<span>4<br>AAA 005</span>',
+                 4, '<span>AAA 006</span><br>AAA 00<span>7</span><br>AAA 00<span>8</span><br>AAA 00<span>9</span><br>AAA <span>1</span>00')]
+    assert result == expected
+
--- a/MoinMoin/util/_tests/test_diff_text.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_diff_text.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,7 +8,7 @@
 
 from MoinMoin.util import diff_text
 
-class TestDiffText:
+class TestDiffText(object):
 
     def testDiff(self):
         """ util.diff_text.diff: test correct diff calculation """
--- a/MoinMoin/util/_tests/test_filesys.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_filesys.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,11 +8,14 @@
 import sys, os, time
 import shutil, tempfile
 
-import py.test
+import pytest
 
 from MoinMoin.util import filesys
 
-class TestFuid:
+win32_only = pytest.mark.skipif("sys.platform != 'win32'")
+win32_incompatible = pytest.mark.skipif("sys.platform == 'win32'")
+
+class TestFuid(object):
     """ test filesys.fuid (a better mtime() alternative for up-to-date checking) """
 
     def setup_method(self, method):
@@ -51,11 +54,10 @@
 
         assert uid2 != uid1 # we changed size and maybe mtime
 
+    @win32_incompatible
     def testUpdateFileMovingFromTemp(self):
         # update file by moving another file over it
         # changing inode, maybe mtime, but not size
-        if sys.platform == 'win32':
-            py.test.skip("Inode change detection not supported on win32")
 
         self.makefile(self.fname, "foo")
         uid1 = filesys.fuid(self.fname)
@@ -66,10 +68,9 @@
 
         assert uid2 != uid1 # we didn't change size, but inode and maybe mtime
 
+    @win32_only
     def testStale(self):
         # is a file with mtime older than max_staleness considered stale?
-        if sys.platform != 'win32':
-            py.test.skip("max_staleness check only done on win32 because it doesn't support inode change detection")
 
         self.makefile(self.fname, "foo")
         uid1 = filesys.fuid(self.fname)
@@ -79,7 +80,7 @@
         assert uid2 != uid1  # should be considered stale if platform has no inode support
 
 
-class TestRename:
+class TestRename(object):
     """ test filesys.rename* """
 
     def setup_method(self, method):
@@ -109,12 +110,12 @@
         self.makefile(self.dst, "dst")
         # win32-like rename does not overwrite an existing destination
         # (on posix, we emulate this behaviour)
-        py.test.raises(OSError, filesys.rename_no_overwrite, self.src, self.dst)
+        pytest.raises(OSError, filesys.rename_no_overwrite, self.src, self.dst)
 
     def test_special_rename_exists(self):
         self.makefile(self.src, "src")
         self.makefile(self.dst, "dst")
-        py.test.raises(OSError, filesys.rename_no_overwrite, self.src, self.dst, delete_old=True)
+        pytest.raises(OSError, filesys.rename_no_overwrite, self.src, self.dst, delete_old=True)
         assert not os.path.exists(self.src)
 
     def test_posix_rename_notexists(self):
@@ -135,4 +136,35 @@
         assert not os.path.exists(self.src)
 
 
+class TestCopy(object):
+    """test filesys.copytree"""
+
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'copytree1')
+        self.src1 = os.path.join(self.test_dir, "copytree-src1")
+        self.src2 = os.path.join(self.test_dir, "copytree-src2")
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+        shutil.rmtree(self.test_dest_dir)
+
+    def makefile(self, src, content):
+        f = open(src, "w")
+        f.write(content)
+        f.close()
+
+    def test_copytree(self):
+        self.makefile(self.src1, 'src1')
+        self.makefile(self.src2, 'src2')
+        self.test_dest_dir = self.test_dir + '_copy'
+        filesys.copytree(self.test_dir, self.test_dest_dir)
+        # check for the dir contents
+        assert os.listdir(self.test_dir) == os.listdir(self.test_dest_dir)
+
+    def test_dir_exist(self):
+        """ raise Error if dir already exist """
+        self.test_dest_dir = tempfile.mkdtemp('', 'temp_dir')
+        with pytest.raises(OSError):
+            filesys.copytree(self.test_dir, self.test_dest_dir)
+
 coverage_modules = ['MoinMoin.util.filesys']
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_forms.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - MoinMoin.util.forms Tests
+"""
+
+from MoinMoin.util import forms
+import pytest
+
+class Bind(object):
+    """ class for self defined test_bind attributes """
+    def __init__(self):
+        self.label = 'test_content'
+        self.default_value = 'test_bind_default'
+        self.optional = True
+        self.properties = {'autofocus': False, 'placeholder': None}
+        self.errors = None
+test_bind = Bind()
+
+test_attributes = {'type': 'submit', u'required': 'test_required', 'autofocus': None, 'placeholder': None}
+
+def test_label_filter():
+    # when content is None
+    result1 = forms.label_filter('test_tagname', test_attributes, None, 'test_context', test_bind)
+    expected = 'test_content'
+    assert result1 == expected
+
+    # when content is not None
+    result2 = forms.label_filter('test_tagname', test_attributes, 'new_content', 'test_context', test_bind)
+    expected = 'new_content'
+    assert result2 == expected
+
+def test_button_filter():
+    result = forms.button_filter('test_tagname', test_attributes, 'new_content', 'test_context', None)
+    expected = 'new_content'
+    assert result == expected
+
+    # attributes.get('type') in ['submit', 'reset', ])
+    content_result = forms.button_filter('input', test_attributes, 'new_content', 'test_context', test_bind)
+    expected = 'new_content'
+    assert content_result == expected
+    attributes_result = test_attributes['value']
+    expected = 'test_bind_default'
+    assert attributes_result == expected
+
+    # tagname == 'button'
+    content_result = forms.button_filter('button', test_attributes, None, 'test_context', test_bind)
+    expected = 'test_bind_default'
+    assert content_result == expected
+
+def test_required_filter():
+    test_bind.optional = False
+    test_attributes[u'class'] = 'test_class'
+    content_result = forms.required_filter('test_tagname', test_attributes, 'new_content', 'test_context', test_bind)
+    expected = 'new_content'
+    assert content_result == expected
+    attribute_result = test_attributes[u'class']
+    expected = u'required'
+    assert attribute_result == expected
+
+    # tagname == 'input'
+    content_result = forms.required_filter('input', test_attributes, 'new_content', 'test_context', test_bind)
+    expected = 'new_content'
+    assert content_result == expected
+    attribute_result = test_attributes[u'required']
+    expected = u'required'
+    assert attribute_result == expected
+
+def test_autofocus_filter():
+    test_bind.properties = {'autofocus': True}
+    content_result = forms.autofocus_filter('test_tagname', test_attributes, 'new_content', 'test_context', test_bind)
+    assert content_result == 'new_content'
+    attribute_result = test_attributes[u'autofocus']
+    assert attribute_result == u'autofocus'
+
+def test_placeholder_filter():
+    test_bind.properties['placeholder'] = 'test_placeholder'
+    content_result = forms.placeholder_filter('test_tagname', test_attributes, 'new_content', 'test_context', test_bind)
+    assert content_result == 'new_content'
+    attribute_result = test_attributes['placeholder']
+    assert attribute_result == 'test_placeholder'
+
+def test_error_filter_factory():
+    # when 'class' not in test_attributes
+    test_bind.errors = 'test_errors'
+    test_attributes.pop(u'class')
+    test_fun_returned = forms.error_filter_factory('test_moin_error')
+    content_result = test_fun_returned('test_tagname', test_attributes, 'new_content', 'test_context', test_bind)
+    assert content_result == 'new_content'
+    attribute_result = test_attributes['class']
+    assert attribute_result == 'test_moin_error'
+
+    # class in test_attributes
+    test_attributes['class'] = 'test_attribute_class'
+    content_result = test_fun_returned('test_tagname', test_attributes, 'new_content', 'test_context', test_bind)
+    assert content_result == 'new_content'
+    attribute_result = test_attributes['class']
+    expected = 'test_attribute_class test_moin_error'
+    assert attribute_result == expected
+
--- a/MoinMoin/util/_tests/test_interwiki.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_interwiki.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,7 +9,7 @@
 
 from __future__ import absolute_import, division
 
-import py
+import pytest
 import tempfile
 import os.path
 import shutil
@@ -54,7 +54,7 @@
         tmpdir = tempfile.mkdtemp()
 
         # test an invalid file
-        with py.test.raises(IOError):
+        with pytest.raises(IOError):
             InterWikiMap.from_file(os.path.join(tmpdir, 'void'))
 
         # test a consistent valid file
@@ -71,7 +71,7 @@
         with open(testfile, 'w') as f:
             f.write('# This is a malformed interwiki file\n'
                     'fails # ever')
-        with py.test.raises(ValueError):
+        with pytest.raises(ValueError):
             InterWikiMap.from_file(testfile)
 
         # finally destroy everything
@@ -99,7 +99,7 @@
                 dict(link1='http://link1.com/',
                      link2='http://link2.in/'))
         # test invalid strings
-        with py.test.raises(ValueError):
+        with pytest.raises(ValueError):
             InterWikiMap.from_string(u'foobarbaz')
 
 
--- a/MoinMoin/util/_tests/test_iri.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_iri.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,7 +7,7 @@
 """
 
 
-import py.test
+import pytest
 
 from MoinMoin.util.iri import *
 
--- a/MoinMoin/util/_tests/test_lock.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_lock.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,9 +9,9 @@
 
 import tempfile, os, time, shutil
 
-import py
+import pytest
 
-from MoinMoin.util.lock import ExclusiveLock
+from MoinMoin.util.lock import ExclusiveLock, WriteLock, ReadLock
 
 
 class TestExclusiveLock(object):
@@ -43,7 +43,7 @@
 
     def testTimeout(self):
         """ util.lock: ExclusiveLock: raise ValueError for timeout < 2.0 """
-        py.test.raises(ValueError, ExclusiveLock, self.lock_dir, timeout=1.0)
+        pytest.raises(ValueError, ExclusiveLock, self.lock_dir, timeout=1.0)
 
     def testAcquire(self):
         """ util.lock: ExclusiveLock: acquire """
@@ -57,7 +57,7 @@
         """
         lock = ExclusiveLock(self.lock_dir)
         if not lock.acquire(0.1):
-            py.test.skip("can't acquire lock")
+            pytest.skip("can't acquire lock")
         lock.release()
         assert lock.acquire(0.1)
 
@@ -65,7 +65,7 @@
         """ util.lock: ExclusiveLock: isLocked """
         lock = ExclusiveLock(self.lock_dir)
         if not lock.acquire(0.1):
-            py.test.skip("can't acquire lock")
+            pytest.skip("can't acquire lock")
         assert lock.isLocked()
         lock.release()
         assert not lock.isLocked()
@@ -74,7 +74,7 @@
         """ util.lock: ExclusiveLock: exists """
         lock = ExclusiveLock(self.lock_dir)
         if not lock.acquire(0.1):
-            py.test.skip("can't acquire lock")
+            pytest.skip("can't acquire lock")
         assert lock.exists()
 
     def testIsExpired(self):
@@ -82,7 +82,7 @@
         timeout = 2.0
         lock = ExclusiveLock(self.lock_dir, timeout=timeout)
         if not lock.acquire(0.1):
-            py.test.skip("can't acquire lock")
+            pytest.skip("can't acquire lock")
         assert not lock.isExpired()
         time.sleep(timeout)
         assert lock.isExpired()
@@ -92,7 +92,7 @@
         timeout = 2.0
         lock = ExclusiveLock(self.lock_dir, timeout=timeout)
         if not lock.acquire(0.1):
-            py.test.skip("can't acquire lock")
+            pytest.skip("can't acquire lock")
         assert not lock.expire()
         time.sleep(timeout)
         assert lock.expire()
@@ -102,7 +102,7 @@
         first = ExclusiveLock(self.lock_dir)
         second = ExclusiveLock(self.lock_dir)
         if not first.acquire(0.1):
-            py.test.skip("can't acquire lock")
+            pytest.skip("can't acquire lock")
         assert not second.acquire(0.1)
 
     def testAcquireAfterTimeout(self):
@@ -114,9 +114,9 @@
         first = ExclusiveLock(self.lock_dir, timeout)
         second = ExclusiveLock(self.lock_dir, timeout)
         if not first.acquire(0.1):
-            py.test.skip("can't acquire lock")
+            pytest.skip("can't acquire lock")
         if second.acquire(0.1):
-            py.test.skip("first lock is not exclusive")
+            pytest.skip("first lock is not exclusive")
         # Second lock should be acquired after timeout
         assert second.acquire(timeout + 0.2)
 
@@ -124,5 +124,49 @@
         time.sleep(delay)
         lock.release()
 
+class TestWriteLock(object):
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'lock_')
+        self.lock_dir = os.path.join(self.test_dir, "writelock")
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+
+    def test_writelock_acquire(self):
+        """ util.lock: WriteLock: acquire """
+        lock = WriteLock(self.lock_dir)
+        assert lock.acquire(0.1)
+        with pytest.raises(RuntimeError):
+            assert lock.acquire(0.1)
+
+    def test_haveReadLocks(self):
+        """check if there is a ReadLock """
+        timeout = 2.0
+        write_lock = WriteLock(self.lock_dir, timeout)
+        read_lock = ReadLock(self.lock_dir)
+        # acquired ReadLock
+        assert read_lock.acquire(0.1)
+        result_before = write_lock._haveReadLocks()
+        assert result_before
+        # try to acquire WriteLock
+        assert write_lock.acquire()
+        result_after = write_lock._haveReadLocks()
+        assert not result_after
+
+class TestReadLock(object):
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'lock_')
+        self.lock_dir = os.path.join(self.test_dir, "readlock")
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+
+    def test_readlock_acquire(self):
+        """ util.lock: ReadLock: acquire """
+        lock = ReadLock(self.lock_dir)
+        assert lock.acquire(0.1)
+        with pytest.raises(RuntimeError):
+            assert lock.acquire(0.1)
+
 coverage_modules = ['MoinMoin.util.lock']
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_md5crypt.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - MoinMoin.util.md5crypt Tests
+"""
+
+
+import pytest
+from MoinMoin.util import md5crypt
+
+def test_unix_md5_crypt():
+    # when magic != None
+    result = md5crypt.unix_md5_crypt('test_pass', 'Moin_test', '$test_magic$')
+    expected = '$test_magic$Moin_tes$JRfmeHgnmCVhVYW.bTtiY1'
+    assert result == expected
+
+    # when magic == None
+    result = md5crypt.unix_md5_crypt('test_pass', 'Moin_test', None)
+    expected = '$1$Moin_tes$hArc67BzmDWtyWWKO5uxQ1'
+    assert result == expected
+
+def test_apache_md5_crypt():
+    # Here magic == '$apr1$'
+    result = md5crypt.apache_md5_crypt('test_pass', 'Moin_test')
+    expected = '$apr1$Moin_tes$4/5zV8nADrNv3BJcY1rZX1'
+    assert result == expected
+
--- a/MoinMoin/util/_tests/test_mime.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_mime.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 from MoinMoin.util.mime import *
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_mimetype.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - MoinMoin.util.mimetype Tests
+"""
+
+import pytest
+from MoinMoin.util import mimetype
+
+class TestMimeType(object):
+    """ Test: util.mimetype """
+
+    def test_parse_format(self):
+        MimeType_obj = mimetype.MimeType(filename = 'test_file.jpg')
+        # format in config.parser_text_mimetype
+        test = [
+        #test_format    # test_mimetype
+        ('html',        ('text', 'html')),
+        ('css',         ('text', 'css')),
+        ('python',      ('text', 'python')),
+        ('latex',       ('text', 'latex'))
+        ]
+
+        for test_format, test_mimetype in test:
+            result = MimeType_obj.parse_format(test_format)
+            assert result == test_mimetype
+
+        # format not in config.parser_text_mimetype
+        test = [
+        # test_format   # test_mimetype
+        ('wiki',        ('text', 'x.moin.wiki')),
+        ('irc',         ('text', 'irssi')),
+        ('test_random', ('text', 'x-test_random'))
+        ]
+
+        for test_format, test_mimetype in test:
+            result = MimeType_obj.parse_format(test_format)
+            assert result == test_mimetype
+        result = MimeType_obj.parse_format(test_format)
+
+    def test_mime_type(self):
+        test = [
+        # test_extension     # test_major/minor
+        ('.mpeg',            'video/mpeg'),
+        ('.pdf',             'application/pdf'),
+        ('.txt',             'text/plain'),
+        ('.jpeg',            'image/jpeg'),
+        ('',                 'application/octet-stream')
+        ]
+
+        # when mimestr is None
+        for test_extension, test_major_minor in test:
+            MimeType_obj = mimetype.MimeType(filename = 'test_file' + test_extension)
+            result = MimeType_obj.mime_type()
+            expected = test_major_minor
+            assert result == expected
+
+        # when mimestr is not None
+        MimeType_obj = mimetype.MimeType(filename = 'test_file', mimestr = 'image/jpeg;charset="utf-8";misc=moin_misc')
+        result = MimeType_obj.mime_type()
+        assert result == 'image/jpeg'
+
+    def test_content_type(self):
+        MimeType_obj = mimetype.MimeType('test_file.mpeg')
+
+        result1 = MimeType_obj.content_type(major = 'application', minor = 'pdf', charset="utf-16", params=None)
+        expected = 'application/pdf'
+        assert result1 == expected
+
+        # major == 'text'
+        result2 = MimeType_obj.content_type(major = 'text', minor = 'plain', charset="utf-16", params=None)
+        expected = 'text/plain; charset="utf-16"'
+        assert result2 == expected
+
+        # when all the parameters passed are None
+        result3 = MimeType_obj.content_type()
+        expected = 'text/x-test_file.mpeg; charset="utf-8"'
+        assert result3 == expected
+
--- a/MoinMoin/util/_tests/test_paramparser.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_paramparser.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py
+import pytest
 
 from MoinMoin.util import paramparser
 
@@ -139,9 +139,9 @@
         result = paramparser.UnitArgument('7mm', float, ['%', 'mm'])
         assert result.get_default() ==  (7.0, 'mm')
         assert result.parse_argument('8%') == (8.0, '%')
-        py.test.raises(ValueError, result.parse_argument,  u'7m')
-        py.test.raises(ValueError, result.parse_argument,  u'7')
-        py.test.raises(ValueError, result.parse_argument,  u'mm')
+        pytest.raises(ValueError, result.parse_argument,  u'7m')
+        pytest.raises(ValueError, result.parse_argument,  u'7')
+        pytest.raises(ValueError, result.parse_argument,  u'mm')
 
     def testExtendedParser(self):
         tests = [
@@ -312,7 +312,7 @@
         ]
 
         def _check(args, sep, kwsep, err):
-            py.test.raises(err,
+            pytest.raises(err,
                            paramparser.parse_quoted_separated_ext,
                            args, sep, kwsep,
                            brackets=(u'<>', u'()'))
@@ -345,22 +345,22 @@
 
     def testGetBooleanRaising(self):
         # wrong default type
-        py.test.raises(AssertionError, paramparser.get_bool, None, None, 42)
+        pytest.raises(AssertionError, paramparser.get_bool, None, None, 42)
 
         # anything except None or unicode raises TypeError
-        py.test.raises(TypeError, paramparser.get_bool, True)
-        py.test.raises(TypeError, paramparser.get_bool, 42)
-        py.test.raises(TypeError, paramparser.get_bool, 42.0)
-        py.test.raises(TypeError, paramparser.get_bool, '')
-        py.test.raises(TypeError, paramparser.get_bool, tuple())
-        py.test.raises(TypeError, paramparser.get_bool, [])
-        py.test.raises(TypeError, paramparser.get_bool, {})
+        pytest.raises(TypeError, paramparser.get_bool, True)
+        pytest.raises(TypeError, paramparser.get_bool, 42)
+        pytest.raises(TypeError, paramparser.get_bool, 42.0)
+        pytest.raises(TypeError, paramparser.get_bool, '')
+        pytest.raises(TypeError, paramparser.get_bool, tuple())
+        pytest.raises(TypeError, paramparser.get_bool, [])
+        pytest.raises(TypeError, paramparser.get_bool, {})
 
         # any value not convertable to boolean raises ValueError
-        py.test.raises(ValueError, paramparser.get_bool, u'')
-        py.test.raises(ValueError, paramparser.get_bool, u'42')
-        py.test.raises(ValueError, paramparser.get_bool, u'wrong')
-        py.test.raises(ValueError, paramparser.get_bool, u'"True"') # must not be quoted!
+        pytest.raises(ValueError, paramparser.get_bool, u'')
+        pytest.raises(ValueError, paramparser.get_bool, u'42')
+        pytest.raises(ValueError, paramparser.get_bool, u'wrong')
+        pytest.raises(ValueError, paramparser.get_bool, u'"True"') # must not be quoted!
 
     def testGetInt(self):
         tests = [
@@ -379,22 +379,22 @@
 
     def testGetIntRaising(self):
         # wrong default type
-        py.test.raises(AssertionError, paramparser.get_int, None, None, 42.23)
+        pytest.raises(AssertionError, paramparser.get_int, None, None, 42.23)
 
         # anything except None or unicode raises TypeError
-        py.test.raises(TypeError, paramparser.get_int, True)
-        py.test.raises(TypeError, paramparser.get_int, 42)
-        py.test.raises(TypeError, paramparser.get_int, 42.0)
-        py.test.raises(TypeError, paramparser.get_int, '')
-        py.test.raises(TypeError, paramparser.get_int, tuple())
-        py.test.raises(TypeError, paramparser.get_int, [])
-        py.test.raises(TypeError, paramparser.get_int, {})
+        pytest.raises(TypeError, paramparser.get_int, True)
+        pytest.raises(TypeError, paramparser.get_int, 42)
+        pytest.raises(TypeError, paramparser.get_int, 42.0)
+        pytest.raises(TypeError, paramparser.get_int, '')
+        pytest.raises(TypeError, paramparser.get_int, tuple())
+        pytest.raises(TypeError, paramparser.get_int, [])
+        pytest.raises(TypeError, paramparser.get_int, {})
 
         # any value not convertable to int raises ValueError
-        py.test.raises(ValueError, paramparser.get_int, u'')
-        py.test.raises(ValueError, paramparser.get_int, u'23.42')
-        py.test.raises(ValueError, paramparser.get_int, u'wrong')
-        py.test.raises(ValueError, paramparser.get_int, u'"4711"') # must not be quoted!
+        pytest.raises(ValueError, paramparser.get_int, u'')
+        pytest.raises(ValueError, paramparser.get_int, u'23.42')
+        pytest.raises(ValueError, paramparser.get_int, u'wrong')
+        pytest.raises(ValueError, paramparser.get_int, u'"4711"') # must not be quoted!
 
     def testGetFloat(self):
         tests = [
@@ -415,21 +415,21 @@
 
     def testGetFloatRaising(self):
         # wrong default type
-        py.test.raises(AssertionError, paramparser.get_float, None, None, u'42')
+        pytest.raises(AssertionError, paramparser.get_float, None, None, u'42')
 
         # anything except None or unicode raises TypeError
-        py.test.raises(TypeError, paramparser.get_float, True)
-        py.test.raises(TypeError, paramparser.get_float, 42)
-        py.test.raises(TypeError, paramparser.get_float, 42.0)
-        py.test.raises(TypeError, paramparser.get_float, '')
-        py.test.raises(TypeError, paramparser.get_float, tuple())
-        py.test.raises(TypeError, paramparser.get_float, [])
-        py.test.raises(TypeError, paramparser.get_float, {})
+        pytest.raises(TypeError, paramparser.get_float, True)
+        pytest.raises(TypeError, paramparser.get_float, 42)
+        pytest.raises(TypeError, paramparser.get_float, 42.0)
+        pytest.raises(TypeError, paramparser.get_float, '')
+        pytest.raises(TypeError, paramparser.get_float, tuple())
+        pytest.raises(TypeError, paramparser.get_float, [])
+        pytest.raises(TypeError, paramparser.get_float, {})
 
         # any value not convertable to int raises ValueError
-        py.test.raises(ValueError, paramparser.get_float, u'')
-        py.test.raises(ValueError, paramparser.get_float, u'wrong')
-        py.test.raises(ValueError, paramparser.get_float, u'"47.11"') # must not be quoted!
+        pytest.raises(ValueError, paramparser.get_float, u'')
+        pytest.raises(ValueError, paramparser.get_float, u'wrong')
+        pytest.raises(ValueError, paramparser.get_float, u'"47.11"') # must not be quoted!
 
     def testGetComplex(self):
         tests = [
@@ -458,25 +458,25 @@
 
     def testGetComplexRaising(self):
         # wrong default type
-        py.test.raises(AssertionError, paramparser.get_complex, None, None, u'42')
+        pytest.raises(AssertionError, paramparser.get_complex, None, None, u'42')
 
         # anything except None or unicode raises TypeError
-        py.test.raises(TypeError, paramparser.get_complex, True)
-        py.test.raises(TypeError, paramparser.get_complex, 42)
-        py.test.raises(TypeError, paramparser.get_complex, 42.0)
-        py.test.raises(TypeError, paramparser.get_complex, 3j)
-        py.test.raises(TypeError, paramparser.get_complex, '')
-        py.test.raises(TypeError, paramparser.get_complex, tuple())
-        py.test.raises(TypeError, paramparser.get_complex, [])
-        py.test.raises(TypeError, paramparser.get_complex, {})
+        pytest.raises(TypeError, paramparser.get_complex, True)
+        pytest.raises(TypeError, paramparser.get_complex, 42)
+        pytest.raises(TypeError, paramparser.get_complex, 42.0)
+        pytest.raises(TypeError, paramparser.get_complex, 3j)
+        pytest.raises(TypeError, paramparser.get_complex, '')
+        pytest.raises(TypeError, paramparser.get_complex, tuple())
+        pytest.raises(TypeError, paramparser.get_complex, [])
+        pytest.raises(TypeError, paramparser.get_complex, {})
 
         # any value not convertable to int raises ValueError
-        py.test.raises(ValueError, paramparser.get_complex, u'')
-        py.test.raises(ValueError, paramparser.get_complex, u'3jj')
-        py.test.raises(ValueError, paramparser.get_complex, u'3Ij')
-        py.test.raises(ValueError, paramparser.get_complex, u'3i-3i')
-        py.test.raises(ValueError, paramparser.get_complex, u'wrong')
-        py.test.raises(ValueError, paramparser.get_complex, u'"47.11"') # must not be quoted!
+        pytest.raises(ValueError, paramparser.get_complex, u'')
+        pytest.raises(ValueError, paramparser.get_complex, u'3jj')
+        pytest.raises(ValueError, paramparser.get_complex, u'3Ij')
+        pytest.raises(ValueError, paramparser.get_complex, u'3i-3i')
+        pytest.raises(ValueError, paramparser.get_complex, u'wrong')
+        pytest.raises(ValueError, paramparser.get_complex, u'"47.11"') # must not be quoted!
 
     def testGetUnicode(self):
         tests = [
@@ -495,16 +495,16 @@
 
     def testGetUnicodeRaising(self):
         # wrong default type
-        py.test.raises(AssertionError, paramparser.get_unicode, None, None, 42)
+        pytest.raises(AssertionError, paramparser.get_unicode, None, None, 42)
 
         # anything except None or unicode raises TypeError
-        py.test.raises(TypeError, paramparser.get_unicode, True)
-        py.test.raises(TypeError, paramparser.get_unicode, 42)
-        py.test.raises(TypeError, paramparser.get_unicode, 42.0)
-        py.test.raises(TypeError, paramparser.get_unicode, '')
-        py.test.raises(TypeError, paramparser.get_unicode, tuple())
-        py.test.raises(TypeError, paramparser.get_unicode, [])
-        py.test.raises(TypeError, paramparser.get_unicode, {})
+        pytest.raises(TypeError, paramparser.get_unicode, True)
+        pytest.raises(TypeError, paramparser.get_unicode, 42)
+        pytest.raises(TypeError, paramparser.get_unicode, 42.0)
+        pytest.raises(TypeError, paramparser.get_unicode, '')
+        pytest.raises(TypeError, paramparser.get_unicode, tuple())
+        pytest.raises(TypeError, paramparser.get_unicode, [])
+        pytest.raises(TypeError, paramparser.get_unicode, {})
 
 
 class TestExtensionInvoking(object):
@@ -562,16 +562,16 @@
         ief(self._test_invoke_int_None, u'i=1')
         ief(self._test_invoke_int_None, u'i=')
         ief(self._test_invoke_int_None, u'')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_int_None, u'x')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_int_None, u'""')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_int_None, u'i=""')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        _test_invoke_int_fixed, u'a=7', [7, 8])
         ief(_test_invoke_int_fixed, u'i=1', [7, 8])
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        _test_invoke_int_fixed, u'i=""', [7, 8])
         ief(_test_invoke_int_fixed, u'i=', [7, 8])
 
@@ -580,20 +580,20 @@
             ief(choicefn, u'choice=a', [7])
             ief(choicefn, u'choice=', [7])
             ief(choicefn, u'choice="a"', [7])
-            py.test.raises(ValueError, ief,
+            pytest.raises(ValueError, ief,
                            choicefn, u'x', [7])
-            py.test.raises(ValueError, ief,
+            pytest.raises(ValueError, ief,
                            choicefn, u'choice=x', [7])
 
         ief(self._test_invoke_float_None, u'i=1.4')
         ief(self._test_invoke_float_None, u'i=')
         ief(self._test_invoke_float_None, u'')
         ief(self._test_invoke_float_None, u'1.4')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_float_None, u'x')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_float_None, u'""')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_float_None, u'i=""')
         ief(self._test_trailing, u'a=7, a')
         ief(self._test_trailing, u'7, a')
@@ -607,15 +607,15 @@
             [{u'7 \xc3': 'test', 'test': u'x'}])
         ief(self._test_arbitrary_kw, u'7 \xc3=test, test= x ',
             [{u'7 \xc3': 'test', 'test': u'x'}])
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_float_required, u'')
         ief(self._test_invoke_float_required, u'1.4')
         ief(self._test_invoke_float_required, u'i=1.4')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_choice_required, u'')
         ief(self._test_invoke_choice_required, u'a')
         ief(self._test_invoke_choice_required, u'i=a')
-        py.test.raises(ValueError, ief,
+        pytest.raises(ValueError, ief,
                        self._test_invoke_float_required, u',')
 
     def testConstructors(self):
@@ -633,13 +633,13 @@
         obj = ief(TEST1, u'a=7')
         assert isinstance(obj, TEST1)
         assert obj.constructed
-        py.test.raises(ValueError, ief, TEST1, u'b')
+        pytest.raises(ValueError, ief, TEST1, u'b')
 
         obj = ief(TEST2, u'a=7')
         assert isinstance(obj, TEST1)
         assert isinstance(obj, TEST2)
         assert obj.constructed
-        py.test.raises(ValueError, ief, TEST2, u'b')
+        pytest.raises(ValueError, ief, TEST2, u'b')
 
         # old style class
         class TEST3:
@@ -653,20 +653,20 @@
         obj = ief(TEST3, u'a=7')
         assert isinstance(obj, TEST3)
         assert obj.constructed
-        py.test.raises(ValueError, ief, TEST3, u'b')
+        pytest.raises(ValueError, ief, TEST3, u'b')
 
         obj = ief(TEST4, u'a=7')
         assert isinstance(obj, TEST3)
         assert isinstance(obj, TEST4)
         assert obj.constructed
-        py.test.raises(ValueError, ief, TEST4, u'b')
+        pytest.raises(ValueError, ief, TEST4, u'b')
 
     def testFailing(self):
         ief = paramparser.invoke_extension_function
 
-        py.test.raises(TypeError, ief, hex, u'15')
-        py.test.raises(TypeError, ief, cmp, u'15')
-        py.test.raises(AttributeError, ief, unicode, u'15')
+        pytest.raises(TypeError, ief, hex, u'15')
+        pytest.raises(TypeError, ief, cmp, u'15')
+        pytest.raises(AttributeError, ief, unicode, u'15')
 
     def testAllDefault(self):
         ief = paramparser.invoke_extension_function
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_pycdb.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - MoinMoin.util.pycdb Tests
+"""
+
+import os
+import shutil, tempfile
+
+import pytest
+from MoinMoin.util import pycdb
+
+class TestCDBMaker(object):
+    """ Test: util.pycdb.CDBMaker """
+
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'test_cdb')
+        self.test_tmpname = os.path.join(self.test_dir, "test_tmpfile")
+        self.test_cdbname = os.path.join(self.test_dir, "test_cdbfile")
+        self.CDBMaker_obj = pycdb.CDBMaker(self.test_cdbname, self.test_tmpname)
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+
+    def test_add(self):
+        result = os.listdir(self.test_dir)
+        result1 = self.CDBMaker_obj.__len__()
+        expected = ['test_tmpfile']
+        assert result == expected
+
+        self.CDBMaker_obj = self.CDBMaker_obj.add(' k_value - ', 'v_value')
+        self.CDBMaker_obj._fp = open(self.test_tmpname, 'r')
+        # seek to 2048 since self._pos was assigned to 2048 initially.
+        self.CDBMaker_obj._fp.seek(2048)
+        # read the contents i.e. newly added contents
+        result = self.CDBMaker_obj._fp.read()
+        expected = '\x0b\x00\x00\x00\x07\x00\x00\x00 k_value - v_value'
+        assert result == expected
+
+    def test_finish(self):
+        # add contents to cdb_file
+        self.CDBMaker_obj = self.CDBMaker_obj.add(' k_value - ', 'v_value')
+        # remove tmpfile
+        self.CDBMaker_obj.finish()
+        result = os.listdir(self.test_dir)
+        expected = ['test_cdbfile']
+        assert result == expected
+
+class TestCDBReader(object):
+    """ Test: util.pycdb.CDBReader """
+
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'test_cdb')
+        self.test_tmpname = os.path.join(self.test_dir, "test_tmpfile")
+        self.test_cdbname = os.path.join(self.test_dir, "test_cdbfile")
+        self.CDBMaker_obj = pycdb.CDBMaker(self.test_cdbname, self.test_tmpname)
+        # add k and v
+        self.CDBMaker_obj = self.CDBMaker_obj.add(' k_value - ', 'v_value')
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+
+    def test_get(self):
+        # remove tmpfile
+        self.CDBMaker_obj.finish()
+
+        CDBReader_obj = pycdb.CDBReader(self.test_cdbname)
+        result = CDBReader_obj.get(' k_value - ', failed=None)
+        expected = 'v_value'
+        assert result == expected
+
+        # invalid key
+        result = CDBReader_obj.get('invalid_key', failed='no_such_value')
+        expected = 'no_such_value'
+        assert result == expected
+
+    def test_keys(self):
+        """ test all key realated functions """
+        # add next value
+        self.CDBMaker_obj = self.CDBMaker_obj.add(' k_value_next - ', 'v_value_next')
+        # remove tmpfile
+        self.CDBMaker_obj.finish()
+
+        CDBReader_obj = pycdb.CDBReader(self.test_cdbname)
+        # test: has_key
+        result = CDBReader_obj.has_key(' k_value - ')
+        assert result
+        # test: invalidkey
+        result = CDBReader_obj.has_key('not_present')
+        assert not result
+
+        # test: firstkey
+        result = CDBReader_obj.firstkey()
+        expected = ' k_value - '
+        assert result == expected
+
+        # test: nextkey
+        result = CDBReader_obj.nextkey()
+        expected = ' k_value_next - '
+        assert result == expected
+
+        # test: iterkeys
+        test_keys = CDBReader_obj.iterkeys()
+        result = []
+        [result.append(key) for key in test_keys]
+        expected = [' k_value - ', ' k_value_next - ']
+        assert expected == result
+
+    def test_itervalues(self):
+        # add next value
+        self.CDBMaker_obj = self.CDBMaker_obj.add(' k_value_next - ', 'v_value_next')
+        # remove tmpfile
+        self.CDBMaker_obj.finish()
+
+        CDBReader_obj = pycdb.CDBReader(self.test_cdbname)
+        test_values = CDBReader_obj.itervalues()
+        result = []
+        [result.append(value) for value in test_values]
+        expected = ['v_value', 'v_value_next']
+        assert expected == result
+
+    def test_iteritems(self):
+        # add next value
+        self.CDBMaker_obj = self.CDBMaker_obj.add(' k_value_next - ', 'v_value_next')
+        # remove tmpfile
+        self.CDBMaker_obj.finish()
+
+        CDBReader_obj = pycdb.CDBReader(self.test_cdbname)
+        test_items = CDBReader_obj.iteritems()
+        result = []
+        [result.append(item) for item in test_items]
+        expected = [(' k_value - ', 'v_value'), (' k_value_next - ', 'v_value_next')]
+        assert expected == result
+
--- a/MoinMoin/util/_tests/test_pysupport.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_pysupport.py	Thu Sep 01 00:16:33 2011 +0200
@@ -9,7 +9,7 @@
 
 import os, errno
 
-import py
+import pytest
 
 from flask import current_app as app
 
@@ -26,12 +26,12 @@
 
     def testNonExistingModule(self):
         """ pysupport: import nonexistent module raises ImportError """
-        py.test.raises(ImportError, pysupport.importName,
+        pytest.raises(ImportError, pysupport.importName,
                        'MoinMoin.util.nonexistent', 'importName')
 
     def testNonExistingAttribute(self):
         """ pysupport: import nonexistent attritbue raises AttributeError """
-        py.test.raises(AttributeError, pysupport.importName,
+        pytest.raises(AttributeError, pysupport.importName,
                        'MoinMoin.util.pysupport', 'nonexistent')
 
     def testExisting(self):
@@ -54,7 +54,7 @@
     def checkPackage(self, path):
         for item in (path, os.path.join(path, '__init__.py')):
             if not os.path.exists(item):
-                py.test.skip("Missing or wrong permissions: %s" % item)
+                pytest.skip("Missing or wrong permissions: %s" % item)
 
     def pluginExists(self):
         return (os.path.exists(self.pluginFilePath('.py')) or
@@ -71,8 +71,8 @@
     def testNonExisting(self):
         """ pysupport: import nonexistent wiki plugin fail """
         if self.pluginExists():
-            py.test.skip('plugin exists: %s' % self.plugin)
-        py.test.raises(plugins.PluginMissingError,
+            pytest.skip('plugin exists: %s' % self.plugin)
+        pytest.raises(plugins.PluginMissingError,
                        plugins.importWikiPlugin,
                            app.cfg, 'parser',
                            self.plugin, 'Parser')
@@ -107,7 +107,7 @@
         """ Create test plugin, skiping if plugin exists """
         if self.pluginExists():
             self.shouldDeleteTestPlugin = False
-            py.test.skip("Won't overwrite existing plugin: %s" % self.plugin)
+            pytest.skip("Won't overwrite existing plugin: %s" % self.plugin)
         self.key = crypto.random_string(32, 'abcdefg')
         data = '''
 # If you find this file in your wiki plugin directory, you can safely
@@ -120,7 +120,7 @@
         try:
             file(self.pluginFilePath('.py'), 'w').write(data)
         except Exception as err:
-            py.test.skip("Can't create test plugin: %s" % str(err))
+            pytest.skip("Can't create test plugin: %s" % str(err))
 
     def deleteTestPlugin(self):
         """ Delete plugin files ignoring missing files errors """
@@ -134,3 +134,4 @@
                     raise
 
 coverage_modules = ['MoinMoin.util.pysupport']
+pytest.main("-x test_pysupport.py")
--- a/MoinMoin/util/_tests/test_registry.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_registry.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 from MoinMoin.util.registry import *
 
@@ -69,6 +69,6 @@
     r.unregister(factory_all)
     assert len(r._entries) == 0
 
-    py.test.raises(ValueError, r.unregister, factory_none)
+    pytest.raises(ValueError, r.unregister, factory_none)
     assert len(r._entries) == 0
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_send_file.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - MoinMoin.util.send_file Tests
+"""
+
+import os
+import tempfile, shutil
+
+from MoinMoin.util import send_file
+import pytest
+
+class TestFuid(object):
+    """ test for send_file """
+
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'test_dir')
+        self.fname = os.path.join(self.test_dir, "test_file")
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+
+    def makefile(self, fname, content):
+        f = open(fname, "w")
+        f.write(content)
+        f.close()
+
+    def test_temptest(self):
+        self.makefile(self.fname, 'test_content')
+        result = send_file.send_file(self.fname, as_attachment = True, conditional = True)
+        expected = '<Response streamed [200 OK]>'
+        assert str(result) == expected
+
+        with pytest.raises(TypeError):
+            send_file.send_file(None, as_attachment = True)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/util/_tests/test_thread_monitor.py	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Copyright: 2011 Prashant Kumar <contactprashantat AT gmail DOT com>
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - MoinMoin.util.thread_monitor Tests
+"""
+
+import shutil, tempfile
+import os
+
+import pytest
+from MoinMoin.util.thread_monitor import Monitor
+
+class TestMonitor(object):
+    """ Tests: Monitor """
+
+    def setup_method(self, method):
+        self.test_dir = tempfile.mkdtemp('', 'test_dump')
+        self.src = os.path.join(self.test_dir, "test_dumpfile")
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.test_dir)
+
+    def test_hook(self):
+        """ tests for hooks """
+        Monitor_test_obj = Monitor()
+        result_inactivated = Monitor_test_obj.hook_enabled()
+        assert not result_inactivated
+        # activate the hook
+        Monitor_test_obj.activate_hook()
+        result_activated = Monitor_test_obj.hook_enabled()
+        assert result_activated
+
+    def test_trigger_dump(self):
+        """ test for trigger_dump """
+        Monitor_test_obj = Monitor()
+        # activate the hook first
+        Monitor_test_obj.activate_hook()
+        f = open(self.src, "w")
+        result = Monitor_test_obj.trigger_dump(f)
+        # read the content of first line
+        f = open(self.src, "r")
+        f.seek(1)
+        assert 'Dumping thread' in f.readline()
+
--- a/MoinMoin/util/_tests/test_tree.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_tree.py	Thu Sep 01 00:16:33 2011 +0200
@@ -6,7 +6,7 @@
 """
 
 
-import py.test
+import pytest
 
 from MoinMoin.util.tree import *
 
--- a/MoinMoin/util/_tests/test_util.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_util.py	Thu Sep 01 00:16:33 2011 +0200
@@ -8,7 +8,7 @@
 
 from MoinMoin import util
 
-class TestUtil:
+class TestUtil(object):
 
     def testRangeList(self):
         """ util.rangelist: test correct function for misc. input values """
--- a/MoinMoin/util/_tests/test_version.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/_tests/test_version.py	Thu Sep 01 00:16:33 2011 +0200
@@ -7,7 +7,7 @@
 
 
 from MoinMoin.util.version import Version
-
+import pytest
 
 class TestVersion(object):
     def test_Version(self):
@@ -53,6 +53,9 @@
         assert Version(1, 2, 3, 'c99') < (1, 2, 4)
         assert Version(1, 2, 4) > (1, 2, 3)
         assert Version(1, 2, 4) > (1, 2, 3, 'c99')
+        # test wrong version string format
+        with pytest.raises(ValueError):
+            assert str(Version(version='wrong_value'))
 
 
 coverage_modules = ['MoinMoin.util.version']
--- a/MoinMoin/util/forms.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/MoinMoin/util/forms.py	Thu Sep 01 00:16:33 2011 +0200
@@ -20,7 +20,6 @@
     return contents
 label_filter.tags = set(['label'])
 
-
 def button_filter(tagname, attributes, contents, context, bind):
     """Show translated text in clickable buttons and submits."""
     if bind is None:
--- a/docs/admin/index.rst	Wed Aug 24 11:29:29 2011 +0200
+++ b/docs/admin/index.rst	Thu Sep 01 00:16:33 2011 +0200
@@ -1,32 +1,42 @@
-====================
-Working with indexes
-====================
+=======
+Indexes
+=======
+
+General
+=======
+moin strongly relies on indexes that accelerate access to item metadata and data.
+Indexes are used internally for many operations like item lookup, history,
+iterating over items, search (also for interactive searches), etc..
+
+So, you need to configure indexing correctly first.
+
+moin will automatically update the index when items are created, updated, deleted,
+destroyed or renamed (via the storage api of moin):
+* when users change content via the user interface
+* when xml is unserialized to load items into the backend
+
+In case you have items changing without usage of the backend api or in case
+your index gets damaged or lost, you need to manually do an index build -
+or moin won't be able to work correctly.
+
 Configuration
 =============
-For correct script working you need ``index_dir`` and ``index_dir_tmp`` in
-your wiki config. They have default values and most likely you don't want to change
+Your wiki config needs ``index_dir`` and ``index_dir_tmp`` to point to different
+directories. They have default values and most likely you don't need to change
 them.
 
 But if you want, try something like::
 
-      index_dir = "/path/to/moin-2.0/wiki/index/"
-      index_dir_tmp = "/path/to/moin-2.0/wiki/tmp_build/"
+      index_dir = "/path/to/moin-2.0/wiki/index"
+      index_dir_tmp = "/path/to/moin-2.0/wiki/tmp_build"
 
 **Note:** Paths MUST BE absolute.
 
-For using one index by multiple wikis (wiki farm) you must set up ``interwikiname``
-parameter in your wiki config:
-
-Example::
 
-        interwikiname = u'MyWiki'
-
-**Note:** For correct working interwikiname must be unique for each wiki.
-
-Offline index manipulation
-==========================
-The main goal of offline index manipulation is to let wiki admin build, update, clean,
-move and monitor state of indexes.
+moin index script reference
+===========================
+You can use the ``moin index`` script to build, update, clean, move and monitor
+indexes.
 
 MoinMoin uses 2 indexes: ``latest-revs`` (index stores only current revisions)
 and ``all-revs`` (index stores all revisions).
@@ -38,32 +48,36 @@
 
 Build
 -----
-Just build fresh indexes using moin backend.
+Index all revisions of all items to the index located in ``index_dir_tmp`` (we
+use this separate location so that index building does not affect the index
+your wiki engine is currently using).
+
+If there is no index at that location yet, a new index will be built there.
+If there is already an index at that location, that index will be extended.
 
 Example::
 
     moin index --for <indexname> --action build
 
-Indexes will be built under ``index_dir_tmp`` so index building happens without
-affecting the index your wiki engine uses currently.
+**Note:** moin won't use this index until you have moved it to ``index_dir``.
+
+Move
+----
+Move indexes from ``index_dir_tmp`` to ``index_dir``.
+
+Example::
+
+    moin index --for <indexname> --action move
 
 Update
 ------
-Update indexes to reflect the current backend contents. Add new stuff, remove
-outdated stuff.
+Update the index located in ``index_dir`` to reflect the current backend
+contents. Add new stuff, remove outdated stuff.
 
 Example::
 
     moin index --for <indexname> --action update
 
-Move
-----
-Moving indexes from ``index_dir_tmp`` to ``index_dir``.
-
-Example::
-
-    moin index --for <indexname> --action move
-
 Clean
 -----
 Create empty index in ``index_dir`` for given index (previous will be erased).
@@ -74,56 +88,87 @@
 
 Show
 ----
-Showing content of index files in human readable form.
-
-**Note:** field length limited to 40 chars.
-
-**Note:** fields without attribute ``stored=True`` are not displayed.
+Show contents of the index located in ``index_dir`` in human readable form.
+This is mostly used for debugging.
 
 Example::
 
     moin index --for indexname --action show
 
-Building wiki farm
-==================
-Wiki farm allows admins create several wikis which share one index. So users
-will be able to search in one wiki and also see results from others.
+**Note:** field length limited to 40 chars.
 
-Before start you must prepair your wiki config.
+**Note:** fields without attribute ``stored=True`` are not displayed.
 
-For example, you have 3 wikis: ``Advertising``, ``Sales``, ``Engineering``
+
+Building an index for a single wiki
+===================================
+Build index at separate place, move it at right place:
+
+     moin index --for both --action build
+     moin index --for both --action move
+
+
+Building an index for a wiki farm
+=================================
+If you run a wiki farm (multiple, but related wikis), you may share the index
+between the farm wikis, so farm wiki users will be able to search in one wiki
+and also see results from the others.
+
+Before start you must prepare your wiki configs.
+
+For example, imagine some company uses 2 farm wikis: ``Sales``, ``Engineering``
 
 So, wiki configs will be looking like 
 
-wikiconfig.py for ``Advertising``::
-
-      index_dir = "/path/to/wiki/index/"
-      index_dir_tmp = "/path/to/wiki/tmp_build/"
-      interwikiname = u"Adverising"
-
-wikiconfig.py for ``Sales``::
-
-      index_dir = "/path/to/wiki/index/"
-      index_dir_tmp = "/path/to/wiki/tmp_build/"
-      interwikiname = u"Sales"
+wiki config for ``Sales``::
 
-wikiconfig.py for ``Engineering``::
-
-      index_dir = "/path/to/wiki/index/"
-      index_dir_tmp = "/path/to/wiki/tmp_build/"
-      interwikiname = u"Engineering"
+      interwikiname = u"Sales"
+      index_dir = "/path/to/wiki/index"
+      index_dir_tmp = "/path/to/wiki/index_tmp"
 
-So, after you defined configs you may start building indexes.
+wiki config for ``Engineering``::
 
-**Note:** Do not build indexes for two or more wikis in parallel, you'll damage
-it or get traceback.
+      interwikiname = u"Engineering"
+      index_dir = "/path/to/wiki/index"
+      index_dir_tmp = "/path/to/wiki/index_tmp"
 
-You must successively build index for each wiki in appropriate virtual env and then
-move indexes from ``index_dir_tmp`` to ``index_dir``::
+Now do the initial index building:
 
-     moin index --for both --action build # in Advertising virtual env
      moin index --for both --action build # in Sales virtual env
      moin index --for both --action build # in Engineering virtual env
      moin index --for both --action move # you can run it from any virtual env
 
-So, after that just run moin and try to search for something.
+Now you should have a shared index for all these wikis.
+
+**Note:** Do not build indexes for multiple wikis in parallel, this is not
+supported.
+
+Building indexes while your wiki is running
+===========================================
+If you want to build an index while your wiki is running, you have to be
+careful not to miss any changes that happen while you build the index.
+
+``moin index --action build`` is made to not interfere with your running wiki.
+So you can run this in parallel without taking your wiki offline.
+Depending on the size of your wiki, index build can take rather long - but it
+doesn't matter as you don't have to take your wiki offline for this.
+
+But: if indexing takes rather long, it can easily happen that content that was
+already put into the index is updated afterwards in the online wiki. So we need
+to do a quick index update while the wiki is offline:
+
+Offline your wiki (or at least make it read-only, so no data in it changes).
+
+``moin index --action move`` to move indexes into place.
+
+``moin index --action update`` to add anything we might have missed otherwise.
+As this is not as much as doing a full index build, this should be rather quick
+(but still: it has to look at every item in your wiki, whether it has been
+updated after the initial index build).
+
+Put your wiki back online again.
+
+**Note:** Indexing puts load onto your server, so if you like to do regular
+index rebuilds, schedule them at some time when your server is not too busy
+otherwise.
+
--- a/docs/admin/install.rst	Wed Aug 24 11:29:29 2011 +0200
+++ b/docs/admin/install.rst	Thu Sep 01 00:16:33 2011 +0200
@@ -73,6 +73,18 @@
 That way, you can have all sorts of Pythons in different virtualenv directories
 within your moin2 workdir.
 
+Loading some items
+------------------
+In case you do not want to have a completely empty wiki, you may want to load
+some items into it. We provide some in `contrib/xml` directory and you can load
+them like this::
+
+ # enter your virtual environment:
+ source env/bin/activate
+
+ # load some example items:
+ moin maint_xml --load --file=contrib/xml/preloaded_items.xml
+
 Installing PIL
 ~~~~~~~~~~~~~~
 For some image processing functions (like resizing, rotating) of moin, you
--- a/docs/admin/upgrade.rst	Wed Aug 24 11:29:29 2011 +0200
+++ b/docs/admin/upgrade.rst	Thu Sep 01 00:16:33 2011 +0200
@@ -43,12 +43,10 @@
 the moin2 sample config (do not just use your 1.9 wikiconfig).
 
 
-Configure moin2 to read moin 1.9 data
--------------------------------------
-moin2 can use the `fs19` storage backend to access your moin 1.9 content
-(pages, attachments and users).
-
-Use a **copy** of the 1.9 content, do not point it at your original data.
+Adjusting the moin2 configuration
+---------------------------------
+It is essential that you adjust the wiki config before you export your 1.9
+data to xml:
 
 Configuration::
 
@@ -66,10 +64,12 @@
     mail_login = ...
     # XXX default_markup must be 'wiki' right now
     page_category_regex = ... # XXX check
-    data_dir = ... # same as in 1.9, user profiles must be in data_dir/user
+
+    # think about which backend you will use in the end and configure
+    # it here (this is NOT the fs19 backend!):
     namespace_mapping = \
         create_simple_mapping(
-            backend_uri='fs19:%s' % data_dir,
+            backend_uri='fs2:/some/path/%%(nsname)s',
             content_acl=dict(before=u'', # acl_rights_before in 1.9
                              default=u'', # acl_rights_default
                              after=u'', # acl_rights_after
@@ -80,33 +80,52 @@
                                   hierarchic=False),
         )
 
-    save_xml = '.../backup.xml'
-    load_xml = None
-
-If you start moin now, it will serialize everything it finds in its backend
-to an xml file.
-
-Keep the xml file (you can use it to try different backend configurations).
-
-
-Writing the data to a moin2 backend
------------------------------------
-Reconfigure moin2 to use the backend you like to use (e.g. fs2 backend)::
+Exporting your moin 1.9 data to an XML file
+-------------------------------------------
+moin2 can use the `fs19` storage backend to access your moin 1.9 content
+(pages, attachments and users). The fs19 backend is a bit more than just
+a backend - it also makes the moin 1.9 data look like moin2 data when
+moin accesses them. To support this, it is essential that you adjust your
+wiki config first, see previous section.
 
-    # use same as you already have, but:
-    backend_uri='fs2:/some/path/%%(nsname)s',
-
-    save_xml = None
-    load_xml = '.../backup.xml'
+Then, use a **copy** of your 1.9 content, do not point moin2 it at your
+original data::
 
-If you start moin2 now, it will unserialize your xml file to fill the
-backend with your data.
-
+    moin maint_xml --moin19data=/your/moin19/data --save --file=moin19.xml
 
-Cleaning up the configuration
------------------------------
-You need to import the xml only once, so after doing that, clean up your config::
+This will serialize all your moin 1.9 data into moin19.xml.
 
-    save_xml = None
-    load_xml = None
+Note: depending on the size of your wiki, this can take rather long and consume
+about the same amount of additional disk space.
 
+Importing the XML file into moin2
+---------------------------------
+Just load moin19.xml into the storage backend you have already configured::
+
+    moin maint_xml --load --file=moin19.xml
+
+Note: depending on the size of your wiki, this can take rather long and consume
+about the same amount of additional disk space.
+
+Testing
+-------
+Just start moin now, it should have your data now.
+
+Try "Index" and "History" views to see what's in there.
+
+Check whether your data is complete and working OK.
+
+If you find issues with data migration from moin 1.9 to 2, please check the
+moin2 issue tracker.
+
+Cleanup
+-------
+If you made a **copy** of your 1.9 content, you can remove that copy now.
+
+Maybe keep the moin19.xml for a while in case you want to try other backends,
+but later you can delete that file.
+
+Make sure you keep all backups of your moin 1.9 installation (code, config,
+data), just for the case you are not happy with moin2 and need to go back for
+some reason.
+
--- a/docs/index.rst	Wed Aug 24 11:29:29 2011 +0200
+++ b/docs/index.rst	Thu Sep 01 00:16:33 2011 +0200
@@ -22,6 +22,7 @@
 
    user/searching
    user/markups
+   user/search
 
 Administrating MoinMoin
 =======================
--- a/docs/intro/features.rst	Wed Aug 24 11:29:29 2011 +0200
+++ b/docs/intro/features.rst	Thu Sep 01 00:16:33 2011 +0200
@@ -83,6 +83,15 @@
 * dump backend contents to xml
 * load backend contents from xml
 
+Search / Indexing
+=================
+* important metadata is put into index
+* content data is converted and put into index
+* fast indexed search, fast internal operations
+* flexible and powerful search queries
+* search current and historical contents
+* using a shared index, find stuff in any farm wiki
+
 User Interface
 ==============
 OO user interface
@@ -144,7 +153,7 @@
 * html5, css, javascript with jquery, svg
 * python
 * flask, flask-cache, flask-babel, flask-themes, flask-script
-* werkzeug, pygments, flatland, blinker, babel, emeraldtree, sqlalchemy, sqlite
+* whoosh, werkzeug, pygments, flatland, blinker, babel, emeraldtree, sqlalchemy, sqlite
 * optional: mercurial, postgresql, mysql
 
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docs/user/search.rst	Thu Sep 01 00:16:33 2011 +0200
@@ -0,0 +1,156 @@
+=====================
+Searching and Finding
+=====================
+
+Entering search queries
+=======================
+
+Usually there is a simple and rather short search query input field offered by
+the theme - if you submit a query from there, it will search in item names and
+content (but only in the current stuff, not in non-current revisions) and display
+the search results to you.
+
+On that search results view, you will get a bigger search query input field
+(e.g. for refining your query) and you may also choose to additionally search
+in non-current revision item revisions (selecting that will search in all
+revisions).
+
+Simple search queries
+=====================
+Just enter one or few simple words into the query input field and hit ``Enter``.
+
+If you give multiple words, it will only find documents containing ALL those
+words ("AND" is the default).
+
+You can use AND (default), OR, NOT to refine your search.
+
+Examples
+--------
+Search for "wiki"::
+
+  wiki
+
+Search for documents containing "wiki" AND "moin"::
+
+  wiki moin
+
+  or (does the same):
+
+  wiki AND moin
+
+Search for documents containing "wiki" OR "moin"::
+
+  wiki OR moin
+
+Search for documents containing "wiki" and NOT "bad"::
+
+  wiki NOT bad
+
+Using wildcards
+===============
+
+If you want to enter word fragments or if you are not sure about spelling or
+word form, use wildcards for the parts you do not know:
+
++----------------+-----------------------------------+
+| Wildcard       | Matches                           |
++----------------+-----------------------------------+
+| ``?``          | one arbitrary character           |
++----------------+-----------------------------------+
+| ``*``          | any count of arbitrary characters |
++----------------+-----------------------------------+
+
+Examples
+--------
+Search for something like wiki, wika, wikb, ...::
+
+  wik?
+
+Search for something like wiki, willi, wi, ...::
+
+  w*i
+
+You can also use it for poor man's language independant word stemming.
+
+Matches on clean, cleaner, cleanest, cleaning, ...::
+
+  clean*
+
+Searching in specific fields
+============================
+
+As long as you do not specify otherwise, moin will search in ``name``,
+``name_exact`` and ``content`` fields.
+
+To specify the field to search in, just use the `fieldname:searchterm` syntax.
+
++-----------------------+-------------------------------------------------------+
+| Field name            | Field value                                           |
++-----------------------+-------------------------------------------------------+
+| ``wikiname``          | wiki name, e.g. ITWiki, EngineeringWiki, SalesWiki    |
++-----------------------+-------------------------------------------------------+
+| ``name``              | document name, e.g. Home, MyWikiPage                  |
++-----------------------+-------------------------------------------------------+
+| ``name_exact``        | same as ``name``, but is not tokenized                |
++-----------------------+-------------------------------------------------------+
+| ``content``           | document contents, e.g. This is some example content. |
++-----------------------+-------------------------------------------------------+
+| ``contenttype``       | document type, e.g. text/plain;charset=utf-8          |
++-----------------------+-------------------------------------------------------+
+| ``tags``              | tags of the document, e.g. important, hard, todo      |
++-----------------------+-------------------------------------------------------+
+| ``language``          | (main) language of the document contents, e.g. en     |
++-----------------------+-------------------------------------------------------+
+| ``mtime``             | document modification (submission) time, 201112312359 |
++-----------------------+-------------------------------------------------------+
+| ``address``           | submitter IP address, e.g. 127.0.0.1                  |
++-----------------------+-------------------------------------------------------+
+| ``hostname``          | submitter DNS name, e.g. foo.example.org              |
++-----------------------+-------------------------------------------------------+
+| ``acl``               | access control list (see below)                       |
++-----------------------+-------------------------------------------------------+
+| ``itemlinks``         | link targets of the document, e.g. OtherItem          |
++-----------------------+-------------------------------------------------------+
+| ``itemtransclusions`` | transclusion targets of the document, e.g. OtherItem  |
++-----------------------+-------------------------------------------------------+
+
+Examples
+--------
+Search in metadata fields::
+
+  contenttype:text
+  contenttype:image/jpeg
+  tags:todo
+  mtime:201108
+  address:127.0.0.1
+  language:en
+  hostname:localhost
+
+Search items with an item ACL that explicitly gives Joe read rights::
+
+  acl:Joe:+read
+
+Limiting search to a specific wiki (in a wiki farm's shared index)::
+
+  wikiname:SomeWiki
+
+Notes
+=====
+moin uses indexed search - keep in mind that this has some special properties:
+
+ * as it is using an index, it is rather fast usually
+ * because it is only using the index, it can only find what was put there
+ * if you use wildcards, it will still use the index, but in a different, slower way
+
+E.g.:
+
+ * "foobar" is put into the index somehow
+ * you search for "ooba" - you will not find it, because only "foobar" was put into the index
+ * solution: search for "foobar" - fast and will find it
+ * solution: search for "*ooba*" - slow, but will find it
+
+More infos
+==========
+
+See the `Whoosh query language docs <http://packages.python.org/Whoosh/querylang.html>`_.
+
--- a/setup.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/setup.py	Thu Sep 01 00:16:33 2011 +0200
@@ -76,7 +76,7 @@
     install_requires=[
         'blinker>=1.1', # event signalling (e.g. for change notification trigger)
         'docutils>=0.6', # reST markup processing
-        'Flask>=0.7.1', # micro framework
+        'Flask>=0.7.2', # micro framework
         'Flask-Babel>=0.6', # i18n support
         'Flask-Cache>=0.3.2', # caching support
         'Flask-Script>=0.3', # scripting support
@@ -85,9 +85,9 @@
         'flatland==dev', # repo checkout at revision 269:6c5d262d7eff works
         'Jinja2>=2.5', # template engine
         'pygments>=1.1.1', # src code / text file highlighting
-        'sqlalchemy>=0.6.0', # metadata index and other stuff
+        'sqlalchemy>=0.7.1', # metadata index and other stuff
         'Werkzeug>=0.6.2', # wsgi toolkit
-        'py==1.3.4', # py.test 1.3.4 is needed by unit tests
+        'pytest', # pytest is needed by unit tests
         'whoosh>=2.1.0', # needed for indexed search
         'sphinx', # needed to build the docs
         'pdfminer', # pdf -> text/plain conversion
--- a/wikiconfig.py	Wed Aug 24 11:29:29 2011 +0200
+++ b/wikiconfig.py	Thu Sep 01 00:16:33 2011 +0200
@@ -24,11 +24,6 @@
     data_dir = os.path.join(instance_dir, 'data') # Note: this used to have a trailing / in the past
     index_dir = os.path.join(instance_dir, "index")
     index_dir_tmp = os.path.join(instance_dir, "index_tmp")
-    # This puts the contents from the specified xml file (a serialized backend) into your
-    # backend(s). You can remove this after the first request to your wiki or
-    # from the beginning if you don't want to use this feature at all.
-    load_xml = os.path.join(wikiconfig_dir, 'contrib', 'xml', 'preloaded_items.xml')
-    #save_xml = os.path.join(wikiconfig_dir, 'contrib', 'xml', 'saved_items.xml')
 
     # This provides a simple default setup for your backend configuration.
     # 'fs:' indicates that you want to use the filesystem backend. You can also use