changeset 5044:f5f3d7348564

Xapian2009: Class Xapian.indexing.Index is renamed to XapianIndex. _check_version() was removed, xapian binding check is done by xappy. XapianSearch was moved to Xapian.search. Log message is shown if xapian version is too old.
author Dmitrijs Milajevs <dimazest@gmail.com>
date Thu, 27 Aug 2009 18:46:31 +0300
parents d50ca8a73057
children d77ac3c87ac8
files MoinMoin/events/xapian_index.py MoinMoin/search/Xapian/__init__.py MoinMoin/search/Xapian/_tests/test_wiki_analyzer.py MoinMoin/search/Xapian/indexing.py MoinMoin/search/Xapian/search.py MoinMoin/search/__init__.py MoinMoin/search/_tests/test_search.py MoinMoin/search/_tests/test_wiki_analyzer.py MoinMoin/search/builtin.py MoinMoin/search/queryparser/expressions.py
diffstat 10 files changed, 211 insertions(+), 203 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/events/xapian_index.py	Thu Aug 27 14:58:05 2009 +0200
+++ b/MoinMoin/events/xapian_index.py	Thu Aug 27 18:46:31 2009 +0300
@@ -9,8 +9,8 @@
 
 def _get_index(request):
     try:
-        from MoinMoin.search.Xapian import Index
-        return Index(request)
+        from MoinMoin.search.Xapian import XapianIndex
+        return XapianIndex(request)
     except ImportError:
         pass
 
--- a/MoinMoin/search/Xapian/__init__.py	Thu Aug 27 14:58:05 2009 +0200
+++ b/MoinMoin/search/Xapian/__init__.py	Thu Aug 27 18:46:31 2009 +0300
@@ -7,5 +7,5 @@
     @license: GNU GPL, see COPYING for details.
 """
 
-from MoinMoin.search.Xapian.indexing import Index, Query, UnicodeQuery, MoinSearchConnection, MoinIndexerConnection
+from MoinMoin.search.Xapian.indexing import XapianIndex, Query, MoinSearchConnection, MoinIndexerConnection
 from MoinMoin.search.Xapian.tokenizer import WikiAnalyzer
--- a/MoinMoin/search/Xapian/_tests/test_wiki_analyzer.py	Thu Aug 27 14:58:05 2009 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,108 +0,0 @@
-# -*- coding: iso-8859-1 -*-
-"""
-    MoinMoin - MoinMoin.search.Xapian.tokenizer Tests
-
-    @copyright: 2009 MoinMoin:DmitrijsMilajevs
-    @license: GNU GPL, see COPYING for details.
-"""
-
-
-from MoinMoin._tests import wikiconfig
-
-
-class TestWikiAnalyzer(object):
-
-    word = u'HelpOnMoinTesting'
-    words = {word.lower(): u'',
-             u'help': u'',
-             u'on': u'',
-             u'moin': u'',
-             u'testing': u''}
-
-    def setup_class(self):
-        try:
-            from MoinMoin.search import Xapian
-            self.analyzer = Xapian.WikiAnalyzer(request=self.request, language=self.request.cfg.language_default)
-        except ImportError:
-            py.test.skip('xapian is not installed')
-
-    def test_tokenize(self):
-        words = self.words
-        tokens = list(self.analyzer.tokenize(self.word))
-
-        assert len(tokens) == len(words)
-
-        for token, stemmed in tokens:
-            assert token in words
-            assert words[token] == stemmed
-
-
-class TestWikiAnalyzerStemmed(TestWikiAnalyzer):
-
-    word = u'HelpOnMoinTesting'
-    words = {word.lower(): u'helponmointest',
-             u'help': u'',
-             u'on': u'',
-             u'moin': u'',
-             u'testing': u'test'}
-
-    class Config(wikiconfig.Config):
-
-        xapian_stemming = True
-
-
-class TestWikiAnalyzerSeveralWords(TestWikiAnalyzer):
-
-    word = u'HelpOnMoinTesting OtherWikiWord'
-    words = {u'helponmointesting': u'',
-             u'help': u'',
-             u'on': u'',
-             u'moin': u'',
-             u'testing': u'',
-             u'otherwikiword': u'',
-             u'other': u'',
-             u'wiki': u'',
-             u'word': u''}
-
-
-class TestWikiAnalyzerStemmedSeveralWords(TestWikiAnalyzer):
-
-    word = u'HelpOnMoinTesting OtherWikiWord'
-    words = {u'helponmointesting': u'helponmointest',
-             u'help': u'',
-             u'on': u'',
-             u'moin': u'',
-             u'testing': u'test',
-             u'otherwikiword': u'',
-             u'other': u'',
-             u'wiki': u'',
-             u'word': u''}
-
-    class Config(wikiconfig.Config):
-
-        xapian_stemming = True
-
-
-class TestWikiAnalyzerStemmedHelpOnEditing(TestWikiAnalyzer):
-
-    word = u'HelpOnEditing'
-    words = {u'helponediting': u'helponedit',
-             u'help': u'',
-             u'on': u'',
-             u'editing': u'edit'}
-
-    class Config(wikiconfig.Config):
-
-        xapian_stemming = True
-
-
-class TestWikiAnalyzerStemmedCategoryHomepage(TestWikiAnalyzer):
-
-    word = u'CategoryHomepage'
-    words = {u'categoryhomepage': u'categoryhomepag',
-             u'category': u'categori',
-             u'homepage': u'homepag'}
-
-    class Config(wikiconfig.Config):
-
-        xapian_stemming = True
--- a/MoinMoin/search/Xapian/indexing.py	Thu Aug 27 14:58:05 2009 +0200
+++ b/MoinMoin/search/Xapian/indexing.py	Thu Aug 27 18:46:31 2009 +0300
@@ -119,25 +119,8 @@
         super(StemmedField, self).__init__(name, value)
 
 
-class Index(BaseIndex):
-
-    def __init__(self, request):
-        self._check_version()
-        BaseIndex.__init__(self, request)
+class XapianIndex(BaseIndex):
 
-    def _check_version(self):
-        """ Checks if the correct version of Xapian is installed """
-        # every version greater than or equal to XAPIAN_MIN_VERSION is allowed
-        XAPIAN_MIN_VERSION = (1, 0, 6)
-        major, minor, revision = xapian.major_version(), xapian.minor_version(), xapian.revision()
-        if (major, minor, revision) >= XAPIAN_MIN_VERSION:
-            return
-
-        from MoinMoin.error import ConfigurationError
-        raise ConfigurationError(('MoinMoin needs at least Xapian version '
-                '%d.%d.%d to work correctly. Either disable Xapian '
-                'completetly in your wikiconfig or upgrade your Xapian %d.%d.%d '
-                'installation!') % (XAPIAN_MIN_VERSION + (major, minor, revision)))
 
     def _main_dir(self):
         """ Get the directory of the xapian index """
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/search/Xapian/search.py	Thu Aug 27 18:46:31 2009 +0300
@@ -0,0 +1,81 @@
+# -*- coding: iso-8859-1 -*-
+"""
+    MoinMoin - search engine internals
+
+    @copyright: 2005 MoinMoin:FlorianFesti,
+                2005 MoinMoin:NirSoffer,
+                2005 MoinMoin:AlexanderSchremmer,
+                2006-2008 MoinMoin:ThomasWaldmann,
+                2006 MoinMoin:FranzPletz
+    @license: GNU GPL, see COPYING for details
+"""
+
+
+from MoinMoin import log
+logging = log.getLogger(__name__)
+
+from MoinMoin.search.builtin import BaseSearch, MoinSearch, BaseIndex
+from MoinMoin.search.Xapian.indexing import XapianIndex
+
+class XapianSearch(BaseSearch):
+
+    def __init__(self, request, query, sort='weight', mtime=None, historysearch=0):
+        super(XapianSearch, self).__init__(request, query, sort, mtime, historysearch)
+
+        self.index = self._xapian_index()
+
+    def _xapian_index(self):
+        """ Get the xapian index if possible
+
+        @param request: current request
+        """
+        index = XapianIndex(self.request)
+
+        if index.exists():
+            return index
+
+    def _search(self):
+        """ Search using Xapian
+
+        Get a list of pages using fast xapian search and
+        return moin search in those pages if needed.
+        """
+        clock = self.request.clock
+        pages = None
+        index = self.index
+
+        assert index, 'XXX Assume that index exist, actually we should have thrown an exception, so MoinSearch could be used instead'
+
+        clock.start('_xapianSearch')
+        try:
+            clock.start('_xapianQuery')
+            search_results = index.search(self.query, sort=self.sort, historysearch=self.historysearch)
+            clock.stop('_xapianQuery')
+            logging.debug("_xapianSearch: finds: %r" % search_results)
+        except BaseIndex.LockedException:
+            pass
+
+        # Note: .data is (un)pickled inside xappy, so we get back exactly what
+        #       we had put into it at indexing time (including unicode objects).
+        pages = [{'uid': r.id,
+                  'wikiname': r.data['wikiname'][0],
+                  'pagename': r.data['pagename'][0],
+                  'attachment': r.data['attachment'][0],
+                  'revision': r.data.get('revision', [0])[0]}
+                 for r in search_results]
+        try:
+            if not self.query.xapian_need_postproc():
+                # xapian handled the full query
+                clock.start('_xapianProcess')
+                try:
+                    _ = self.request.getText
+                    return self._getHits(pages), (search_results.estimate_is_exact and '' or _('about'), search_results.matches_estimated)
+                finally:
+                    clock.stop('_xapianProcess')
+        finally:
+            clock.stop('_xapianSearch')
+
+        # some postprocessing by MoinSearch is required
+        return MoinSearch(self.request, self.query, self.sort, self.mtime, self.historysearch, pages=pages)._search()
+
+
--- a/MoinMoin/search/__init__.py	Thu Aug 27 14:58:05 2009 +0200
+++ b/MoinMoin/search/__init__.py	Thu Aug 27 18:46:31 2009 +0300
@@ -14,7 +14,7 @@
 logging = log.getLogger(__name__)
 
 from MoinMoin.search.queryparser import QueryParser, QueryError
-from MoinMoin.search.builtin import MoinSearch, XapianSearch
+from MoinMoin.search.builtin import MoinSearch
 
 
 def searchPages(request, query, sort='weight', mtime=None, historysearch=None, **kw):
@@ -34,10 +34,14 @@
     if isinstance(query, str) or isinstance(query, unicode):
         query = QueryParser(**kw).parse_query(query)
 
+    searcher = MoinSearch
+
     if request.cfg.xapian_search:
-        searcher = XapianSearch
-    else:
-        searcher = MoinSearch
+        try:
+            from MoinMoin.search.Xapian.search import XapianSearch
+            searcher = XapianSearch
+        except ImportError, error:
+            logging.warning("%s. Either disable Xapian completetly in your wikiconfig or upgrade your Xapian installation" % str(error))
 
     return searcher(request, query, sort, mtime=mtime, historysearch=historysearch).run()
 
--- a/MoinMoin/search/_tests/test_search.py	Thu Aug 27 14:58:05 2009 +0200
+++ b/MoinMoin/search/_tests/test_search.py	Thu Aug 27 18:46:31 2009 +0300
@@ -12,7 +12,7 @@
 
 from MoinMoin.search import QueryError
 from MoinMoin.search.queryparser import QueryParser
-from MoinMoin.search.builtin import MoinSearch, XapianSearch
+from MoinMoin.search.builtin import MoinSearch
 from MoinMoin._tests import nuke_xapian_index, wikiconfig, become_trusted, create_page, nuke_page
 from MoinMoin.wikiutil import Version
 
@@ -305,6 +305,7 @@
         xapian_search = True
 
     def get_searcher(self, query):
+        from MoinMoin.search.Xapian.search import XapianSearch
         return XapianSearch(self.request, query)
 
     def get_moin_search_connection(self):
@@ -314,12 +315,16 @@
     def setup_method(self, method):
 
         try:
-            from MoinMoin.search.Xapian import Index
-        except ImportError:
+            from MoinMoin.search.Xapian import XapianIndex
+        except ImportError, error:
+            print str(error)
+            if not str(error).startswith('Xapian '):
+                raise
+
             py.test.skip('xapian is not installed')
 
         nuke_xapian_index(self.request)
-        index = Index(self.request)
+        index = XapianIndex(self.request)
         index.indexPages(mode='add', pages=self.pages)
 
     def teardown_method(self, method):
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/search/_tests/test_wiki_analyzer.py	Thu Aug 27 18:46:31 2009 +0300
@@ -0,0 +1,108 @@
+# -*- coding: iso-8859-1 -*-
+"""
+    MoinMoin - MoinMoin.search.Xapian.tokenizer Tests
+
+    @copyright: 2009 MoinMoin:DmitrijsMilajevs
+    @license: GNU GPL, see COPYING for details.
+"""
+
+import py
+from MoinMoin._tests import wikiconfig
+
+try:
+    from MoinMoin.search.Xapian.tokenizer import WikiAnalyzer
+except ImportError:
+    py.test.skip('xapian is not installed')
+
+class TestWikiAnalyzer(object):
+
+    word = u'HelpOnMoinTesting'
+    words = {word.lower(): u'',
+             u'help': u'',
+             u'on': u'',
+             u'moin': u'',
+             u'testing': u''}
+
+    def setup_class(self):
+        self.analyzer = WikiAnalyzer(request=self.request, language=self.request.cfg.language_default)
+
+    def test_tokenize(self):
+        words = self.words
+        tokens = list(self.analyzer.tokenize(self.word))
+
+        assert len(tokens) == len(words)
+
+        for token, stemmed in tokens:
+            assert token in words
+            assert words[token] == stemmed
+
+
+class TestWikiAnalyzerStemmed(TestWikiAnalyzer):
+
+    word = u'HelpOnMoinTesting'
+    words = {word.lower(): u'helponmointest',
+             u'help': u'',
+             u'on': u'',
+             u'moin': u'',
+             u'testing': u'test'}
+
+    class Config(wikiconfig.Config):
+
+        xapian_stemming = True
+
+
+class TestWikiAnalyzerSeveralWords(TestWikiAnalyzer):
+
+    word = u'HelpOnMoinTesting OtherWikiWord'
+    words = {u'helponmointesting': u'',
+             u'help': u'',
+             u'on': u'',
+             u'moin': u'',
+             u'testing': u'',
+             u'otherwikiword': u'',
+             u'other': u'',
+             u'wiki': u'',
+             u'word': u''}
+
+
+class TestWikiAnalyzerStemmedSeveralWords(TestWikiAnalyzer):
+
+    word = u'HelpOnMoinTesting OtherWikiWord'
+    words = {u'helponmointesting': u'helponmointest',
+             u'help': u'',
+             u'on': u'',
+             u'moin': u'',
+             u'testing': u'test',
+             u'otherwikiword': u'',
+             u'other': u'',
+             u'wiki': u'',
+             u'word': u''}
+
+    class Config(wikiconfig.Config):
+
+        xapian_stemming = True
+
+
+class TestWikiAnalyzerStemmedHelpOnEditing(TestWikiAnalyzer):
+
+    word = u'HelpOnEditing'
+    words = {u'helponediting': u'helponedit',
+             u'help': u'',
+             u'on': u'',
+             u'editing': u'edit'}
+
+    class Config(wikiconfig.Config):
+
+        xapian_stemming = True
+
+
+class TestWikiAnalyzerStemmedCategoryHomepage(TestWikiAnalyzer):
+
+    word = u'CategoryHomepage'
+    words = {u'categoryhomepage': u'categoryhomepag',
+             u'category': u'categori',
+             u'homepage': u'homepag'}
+
+    class Config(wikiconfig.Config):
+
+        xapian_stemming = True
--- a/MoinMoin/search/builtin.py	Thu Aug 27 14:58:05 2009 +0200
+++ b/MoinMoin/search/builtin.py	Thu Aug 27 18:46:31 2009 +0300
@@ -608,68 +608,3 @@
         else:
             return self.request.rootpage.getPageList(user='', exists=0)
 
-
-class XapianSearch(BaseSearch):
-
-    def _xapianIndex(request):
-        """ Get the xapian index if possible
-
-        @param request: current request
-        """
-        try:
-            from MoinMoin.search.Xapian import Index
-            index = Index(request)
-        except ImportError:
-            return None
-
-        if index.exists():
-            return index
-
-    _xapianIndex = staticmethod(_xapianIndex)
-
-    def _search(self):
-        """ Search using Xapian
-
-        Get a list of pages using fast xapian search and
-        return moin search in those pages if needed.
-        """
-        clock = self.request.clock
-        pages = None
-        index = self._xapianIndex(self.request)
-
-        assert index, 'XXX Assume that index exist, actually we should have thrown an exception, so MoinSearch could be used instead'
-
-        clock.start('_xapianSearch')
-        try:
-            clock.start('_xapianQuery')
-            search_results = index.search(self.query, sort=self.sort, historysearch=self.historysearch)
-            clock.stop('_xapianQuery')
-            logging.debug("_xapianSearch: finds: %r" % search_results)
-            self._xapianIndex = index
-        except BaseIndex.LockedException:
-            pass
-
-        # Note: .data is (un)pickled inside xappy, so we get back exactly what
-        #       we had put into it at indexing time (including unicode objects).
-        pages = [{'uid': r.id,
-                  'wikiname': r.data['wikiname'][0],
-                  'pagename': r.data['pagename'][0],
-                  'attachment': r.data['attachment'][0],
-                  'revision': r.data.get('revision', [0])[0]}
-                 for r in search_results]
-        try:
-            if not self.query.xapian_need_postproc():
-                # xapian handled the full query
-                clock.start('_xapianProcess')
-                try:
-                    _ = self.request.getText
-                    return self._getHits(pages), (search_results.estimate_is_exact and '' or _('about'), search_results.matches_estimated)
-                finally:
-                    clock.stop('_xapianProcess')
-        finally:
-            clock.stop('_xapianSearch')
-
-        # some postprocessing by MoinSearch is required
-        return MoinSearch(self.request, self.query, self.sort, self.mtime, self.historysearch, pages=pages)._search()
-
-
--- a/MoinMoin/search/queryparser/expressions.py	Thu Aug 27 14:58:05 2009 +0200
+++ b/MoinMoin/search/queryparser/expressions.py	Thu Aug 27 18:46:31 2009 +0300
@@ -21,7 +21,7 @@
 
 try:
     from MoinMoin.search import Xapian
-    from MoinMoin.search.Xapian import Query, UnicodeQuery
+    from MoinMoin.search.Xapian import Query
 
     OP_AND = Query.OP_AND
     OP_OR = Query.OP_OR