changeset 4980:d43af366f10a

Xapian2009: xapwrap was deleted from MoinMoin/support.
author Dmitrijs Milajevs <dimazest@gmail.com>
date Thu, 06 Aug 2009 17:37:44 +0200
parents d2cda24a5475
children f1d1d8105d52
files MoinMoin/support/xapwrap/__init__.py MoinMoin/support/xapwrap/document.py MoinMoin/support/xapwrap/index.py
diffstat 3 files changed, 0 insertions(+), 1412 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/support/xapwrap/__init__.py	Thu Aug 06 17:26:28 2009 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,2 +0,0 @@
-""" xapwrap version 0.3.1 """
-
--- a/MoinMoin/support/xapwrap/document.py	Thu Aug 06 17:26:28 2009 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,327 +0,0 @@
-"""
-    xapwrap.document - Pythonic wrapper around Xapian's Document API
-"""
-import datetime
-import re
-import cPickle
-import xapian
-
-MAX_KEY_LEN = 240 # this comes from xapian's btree.h, Btree::max_key_len
-# NOTE: xapian's btree.h file says that its actually 252, but due to
-# xapian's implementation details, the actual limit is closer to 245
-# bytes. See http://thread.gmane.org/gmane.comp.search.xapian.cvs/329
-# for more info, especially the second message.
-
-# The limit described above only holds true assuming keys that do not
-# contain any NULL bytes. Since xapian internally escapes \0 bytes,
-# xapian sees the key length as (2*N + 2) where N is the number of
-# embedded NULL characters.
-
-INTER_FIELD_POSITION_GAP = 100
-
-UNICODE_ENCODING = "UTF-8" # XXX this should not be hardcoded on module level
-UNICODE_ERROR_POLICY = "replace"
-
-class StandardAnalyzer:
-    WORD_RE = re.compile('\\w{1,%i}' % MAX_KEY_LEN, re.U)
-
-    def tokenize(self, unknownText):
-        originalText = cleanInputText(unknownText, True)
-        # we want to perform lower() and the re search using a unicode
-        # object. if we try to perform those operations on regular
-        # string object that happens to represent unicode text encoded
-        # with UTF-8, we'll get garbage, or at least an
-        # OS/libc/$LC_CTYPE dependant result
-        text = originalText.lower()
-        for match in self.WORD_RE.finditer(text):
-            # we yield unicode ONLY
-            yield match.group()
-
-
-class TextField(object):
-    __slots__ = ('name', 'text', 'prefix')
-
-    def __init__(self, name, text = '', prefix = False):
-        if name and not text:
-            assert not prefix  # it makes no sense to use a prefixed
-                               # field without a name
-            self.text = name
-            self.name = ''
-        else:
-            self.name = name
-            self.text = text
-        self.prefix = prefix
-
-    def __len__(self):
-        return len(self.text)
-
-class SortKey(object):
-    __slots__ = ('name', 'value', 'index', 'flattener')
-
-    def __init__(self, name, value, index = None, flattener = None):
-        self.name = name
-        self.value = value
-        self.index = index
-        assert (name is None) ^ (index is None)
-        self.flattener = flattener
-
-class Value(SortKey):
-    pass
-
-class Term(object):
-    __slots__ = ('value')
-
-    def __init__(self, value):
-        self.value = value
-
-    def __len__(self):
-        return len(self.value)
-
-class Keyword(object):
-    __slots__ = ('name', 'value')
-
-    def __init__(self, name, value):
-        self.name = name
-        self.value = value
-
-    def __len__(self):
-        return len(self.value)
-
-
-class Document:
-    """
-    @ivar keywords: sequence of Keyword objects
-    @ivar sortFields: sequence of SortKey objects
-    @ivar textFields: sequence of TextField objects
-
-    @cvar analyzerFactory: factory object for constructing analyzers
-    @cvar _picklerProtocol: protocol used in pickling data attributes
-    @cvar _noObject: dummy object used to indicate that there is no
-    data attribute
-    @cvar source: this is an optional argument to point at the
-    original text/object that this document represents
-    """
-    _noObject = object()
-    _picklerProtocol = -1
-    analyzerFactory = StandardAnalyzer
-
-    # XXX TODO: add a fromXapianDoc classmethod that can be used by
-    # indices when returning documents from the db
-
-    def __init__(self, textFields = (), sortFields = (), keywords = (),
-                 terms = (), values = (), uid = None, data = _noObject, source = None):
-        """
-        sortFields and values are really the same thing as far as
-        xapian is concerned. We differentiate them in the hope of
-        making the API easier to understand.
-        """
-        for fields in ('textFields', 'sortFields', 'keywords', 'terms', 'values'):
-            arg = vars()[fields]
-            if not isinstance(arg, (list, tuple)):
-                arg = (arg,)
-            setattr(self, fields, list(arg))
-            # copy the list so we can modify without affecting the original
-        self.uid = uid
-        self.data = data
-        self.source = source
-        # sortFields and values are really the same thing as far as xapian is concerned
-        self.sortFields += self.values
-
-    def __len__(self):
-        length = 0
-        for fieldList in (self.textFields, self.keywords):
-            length += sum(map(len, fieldList))
-
-        if self.data != self._noObject:
-            length += len(cPickle.dumps(self.data, self._picklerProtocol))
-
-        return length
-
-    def toXapianDocument(self, indexValueMap, prefixMap=None):
-        d = xapian.Document()
-        position = 0
-        analyzer = self.analyzerFactory()
-
-        # add text fields
-        for field in self.textFields:
-            # XXX: terms textFields won't get numbered
-            # after each other, needed for titles
-            position = 0
-            for token in analyzer.tokenize(field.text):
-                if isinstance(token, tuple):
-                    token, position = token
-                else:
-                    position += 1
-                # the xapian swig bindings don't like unicode objects, so we
-                # decode terms to UTF-8 before indexing. this is fine as
-                # long as all data that goes into the db (whether for
-                # indexing or search) is converted to UTF-8 string and all
-                # data coming from the db (.get_value(), .get_data()) is
-                # decoded as UTF-8.
-                token = token.encode(UNICODE_ENCODING, UNICODE_ERROR_POLICY)
-                # the tokenizer cannot guarantee that token length is
-                # below MAX_KEY_LEN since the regexp is done with
-                # unicode and the result is later converted to UTF-8. In
-                # the process, the string length could expand, so we
-                # need to check here as well.
-                d.add_posting(checkKeyLen(token), position)
-            #position += INTER_FIELD_POSITION_GAP
-
-            if field.prefix:
-                prefix = field.name
-                for token in analyzer.tokenize(field.text):
-                    if isinstance(token, tuple):
-                        token, position = token
-                    else:
-                        position += 1
-                    # token is unicode, but gets converted to UTF-8
-                    # by makePairForWrite:
-                    term = makePairForWrite(prefix, token, prefixMap)
-                    d.add_posting(term, position)
-                #position += INTER_FIELD_POSITION_GAP
-
-        # add keyword fields
-        for field in self.keywords:
-            term = makePairForWrite(field.name, field.value, prefixMap)
-            d.add_term(term)
-
-        # add non positional terms
-        for term in self.terms:
-            d.add_term(term.value)
-
-        # add sort keys
-        for field in self.sortFields:
-            self.addSortField(d, field, indexValueMap)
-
-        # serialize and add the data object if present
-        if self.data is not self._noObject:
-            dataStr = cPickle.dumps(self.data, self._picklerProtocol)
-            d.set_data(dataStr)
-
-        return d
-
-    def addSortField(self, doc, field, indexValueMap):
-        if field.index is None:
-            valueIndex = indexValueMap.get(field.name, None)
-            if valueIndex is None:
-                from index import NoIndexValueFound
-                raise NoIndexValueFound(field.name, indexValueMap)
-        else:
-            valueIndex = field.index
-        assert isinstance(valueIndex, int)
-
-        if field.flattener:
-            flatValue = field.flattener(field.value)
-        else:
-            flatValue = self.flatten(field.value)
-        # xapian has no limit on value length
-        cleanValue = cleanInputText(flatValue)
-        doc.add_value(valueIndex, cleanValue)
-
-    _flatteners = {}
-
-    def flatten(self, value):
-        t = type(value)
-        if t == str:
-            return value
-        elif t in self._flatteners:
-            flattener = self._flatteners[t]
-            flatVal = flattener(value)
-            return flatVal
-        else:
-            raise ValueError("Cannot flatten %r into a string. Perhaps you "
-                             "should register a flattener for type %r."
-                             % (value, type(value)))
-
-    def registerFlattener(klass, typeToFlatten, flattener):
-        if typeToFlatten in klass._flatteners:
-            raise ValueError("A sort field flattener for type %s has already"
-                             "been registered (%s) but you are attempting to"
-                             "register a new flattener: %s"
-                             % (typeToFlatten, klass._flatteners[typeToFlatten],
-                                flattener))
-        assert callable(flattener)
-        klass._flatteners[typeToFlatten] = flattener
-    registerFlattener = classmethod(registerFlattener)
-
-    def unregisterFlattener(klass, typeToFlatten):
-        if typeToFlatten in klass._flatteners:
-            del klass._flatteners[typeToFlatten]
-    unregisterFlattener = classmethod(unregisterFlattener)
-
-# common flatteners:
-
-def flattenNumeric(value, numDigits = 10):
-    return ''.join(('%', str(numDigits), '.d')) % value
-
-Document.registerFlattener(int, flattenNumeric)
-
-def flattenLong(value):
-    return flattenNumeric(value, numDigits=20)
-
-Document.registerFlattener(long, flattenLong)
-
-def flattenDate(value):
-    return value.isoformat()
-
-for dt in (datetime.date, datetime.time, datetime.datetime):
-    Document.registerFlattener(dt, flattenDate)
-
-def flattenUnicode(value):
-    return value.encode(UNICODE_ENCODING)
-
-Document.registerFlattener(unicode, flattenUnicode)
-
-
-def cleanInputText(unknownText, returnUnicode = False):
-    if isinstance(unknownText, str):
-        originalText = unknownText.decode(UNICODE_ENCODING, UNICODE_ERROR_POLICY) # XXX hardcoded UTF-8, make param XXX
-    elif isinstance(unknownText, unicode):
-        originalText = unknownText
-    else:
-        raise ValueError("Only strings and unicode objects can be indexed.")
-    # be very careful about lowercasing the text here: since the API we
-    # expose to higher levels doesn't allow searchup.py to call
-    # findInField directly, searches for INDEXERVERSION:4 have to be
-    # sent as regular queries. lowercasing all queries here will break
-    # keyword searches.
-    if returnUnicode:
-        return originalText
-    else:
-        return originalText.encode(UNICODE_ENCODING, UNICODE_ERROR_POLICY)
-
-
-def makePairForWrite(prefix, token, prefixMap=None):
-    # prefixes must be uppercase; if the prefix given to us is a str
-    # that happens to be UTF-8 encoded, bad things will happen when we
-    # uppercase it, so we convert everything to unicode first
-    if isinstance(prefix, str):
-        prefix = prefix.decode(UNICODE_ENCODING, UNICODE_ERROR_POLICY)
-    if isinstance(token, str):
-        token = token.decode(UNICODE_ENCODING, UNICODE_ERROR_POLICY) # XXX hardcoded UTF-8, make param
-
-    if prefixMap is None:
-        prefix = prefix.upper()
-    else: # we have a map, so first translate it using the map (e.g. 'title' -> 'S')
-        prefix = prefixMap.get(prefix, prefix.upper())
-
-    result = '%s%s%s' % (prefix, prefix[0] == 'X' and ':' or '', token)
-    # since return value is going into the db, it must be encoded as UTF-8
-    result = result.encode(UNICODE_ENCODING, UNICODE_ERROR_POLICY)
-    return checkKeyLen(result)
-
-def checkKeyLen(s):
-    if not s:
-        return ' '
-    numNullBytes = s.count('\0') + 1
-    xapianLen = numNullBytes + len(s) + 1 # that last one is for the
-                                          # terminating \0
-    if xapianLen < MAX_KEY_LEN:
-        return s
-    else:
-        # doing nothing seems preferable to mangling an overly large
-        # token that we don't know how to handle. we use a space
-        # instead of an empty string because xapian doesn't like
-        # getting empty strings added as terms
-        return ' '
-
--- a/MoinMoin/support/xapwrap/index.py	Thu Aug 06 17:26:28 2009 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,1083 +0,0 @@
-# Copyright (c) 2005 Divmod Inc. See LICENSE file for details.
-"""
-Xapwrap provides an improved interface to the Xapian text indexing
-library (see http://www.xapian.org/ for more information on
-Xapian). Xapwrap provides a layered approach offering ample
-opportunities for customization.
-
-Example
--------
-::
-
-    from xapwrap import SmartIndex, Document, TextField, SortKey
-    from datetime import date
-
-    idx = SmartIndex('/tmp/index', True)
-    d1 = Document(TextField('hi there bob'),
-                  sortFields = [SortKey('date', date(2004, 1, 1)),
-                                SortKey('author', 'Bob'),
-                                SortKey('size', 450)])
-    idx.index(d1)
-    idx.close()
-
-    idx = SmartIndex('/tmp/index')
-    print idx.search('there', 'date', sortAscending = True)
-
-
-
-Indices
--------
-
-Important methods for C{ReadOnlyIndex}:
- __init__(self, *pathnames)
- close(self)
- configure(self, prefixMap = None, indexValueMap = None)
- flush(self)
- search(self, query, sortKeyt = None,
-        startingIndex = 0, batchSize = MAX_DOCS_TO_RETURN,
-        sortIndex = None, sortAscending = True,
-        sortByRelevence = False)
- count(self, query)
- checkIndex(self, maxID)
- get_doccount(self, uid)
-
-Important methods for C{Index}:
- (all methods in ReadOnlyIndex)
- __init__(self, pathname, create)
- index(self, doc)
- add_document(self, doc)
- replace_document(self, uid, doc)
- delete_document(self, uid)
-
-C{SmartIndex} and C{SmartReadOnlyIndex} define the same methods as their
-dumb counterparts.
-
-The primary way to interact with a Xapian index is to use either the
-C{Index} or C{ReadOnlyIndex} class. In addition to offering read only
-access without the inconveniance of lock files, C{ReadOnlyIndex} offers
-the ability to merge several xapian indices into one super index with
-only a small performance impediement.
-
-In addition to C{Index} and C{ReadOnlyIndex}, Xapwrap also offers
-C{SmartIndex} and C{SmartReadOnlyIndex} classes. These classes
-automatically store and manage the index value map and the prefix map in
-the index. There are two caveats to using them however. First, one
-cannot index documents that have a xapian ID of 1. Secondly, when using
-C{SmartReadOnlyIndex} to combine multiple indices together, the indices
-must have consistent value index maps. Indices where all documents have
-the same index value map are always consistent. The problem only emerges
-when indices can have different types of documents with different sets
-of sort keys. More specifically, the problem can only emerge if one
-indices documents in such a way that sort keys are added to different
-indices in different orders.
-
-
-Documents
----------
-
-In order to add new data to an index, one asks a C{Index} or
-C{SmartIndex} instance to index a C{Document} instance. Documents take a
-sequence of text fields, a sequence of sort keys and a sequence of
-keywords as constructor arguments. They also take optional universal
-identifiers and an arbitrary serializable object. The first three
-sequences can be created using the C{TextField}, C{SortKey}, and
-C{Keyword} classes defined below. C{TextField} instances contain a chunk
-of text and an optional name as well as a boolean indicating whether the
-field is to be prefixed. Prefixed fields are effectively indexed twice:
-after being indexed normally, each token is indexed again with the field
-name. This allows the user to perform fielded searches and is primarily
-useful for small text fields, such as the subject of an email or a list
-of author names. C{Keyword} instances denote individual prefixed tokens
-that are indexed with no positional information. C{SortKey} instances
-denote arbitrary fields that are used for sorting documents. They
-include a sort field name and the sort key value. Since Xapian only
-accepts strings as sort keys, sort key values must be flattened into
-strings before entering the index.
-
-Xapwrap defines flattener functions that automatically flatten integer,
-date, time, and datetime instances into strings that sort properly. You
-can define your own flatteners for custom data types by using the
-C{registerFlattener} class method of the C{Document} class.
-
-
-Error Handling
---------------
-Internal Xapian error conditions should generate normal python
-exceptions defined in this file that inherit from xapwrap.XapianError.
-
-
-Logging
--------
-Xapwrap will use twisted's logging facilities if available. In any
-event, a custom logging function can be supplied by setting xapwrap.log.
-
-
-Future Work
------------
-Xapwrap currently does not support stemming or stop words, although a
-future version will.
-
-"""
-try:
-    set
-except:
-    from sets import Set as set
-
-import cPickle, glob, os
-import xapian
-from document import makePairForWrite, StandardAnalyzer, Document, SortKey, Keyword
-from document import UNICODE_ENCODING, UNICODE_ERROR_POLICY
-
-try:
-    from atop.tpython import FilesystemLock
-except ImportError:
-    try:
-        from os import symlink, readlink, remove as rmlink
-    except ImportError:
-        import win32event
-
-        class FilesystemLock:
-            """A mutex
-
-            A real mutex this time. See the non-win32 version for details.
-            """
-
-            locked = False
-            clean = True
-
-            def __init__(self, name):
-                #Mutex name cannot contain backslash
-                name = name.replace('\\', '/')
-                self.name = name
-                self._mutex = win32event.CreateMutex(None, False, name)
-                if not self._mutex:
-                    raise RuntimeError("Failed to create a named mutex")
-
-            def lock(self):
-                res = win32event.WaitForSingleObject(self._mutex, 0)
-                self.locked = (res != win32event.WAIT_TIMEOUT)
-                return self.locked
-
-            def unlock(self):
-                #C API ReleaseMutex version is supposed to return something to
-                #tell whether the lock was correctly released or not. The binding
-                #does not.
-                win32event.ReleaseMutex(self._mutex)
-                self.locked = False
-
-    else:
-        import errno
-
-        class FilesystemLock:
-            """A mutex.
-
-            This relies on the filesystem property that creating
-            a symlink is an atomic operation and that it will
-            fail if the symlink already exists.  Deleting the
-            symlink will release the lock.
-
-            @ivar name: The name of the file associated with this lock.
-            @ivar clean: Indicates whether this lock was released cleanly by its
-            last owner.  Only meaningful after C{lock} has been called and returns
-            True.
-            """
-
-            clean = None
-            locked = False
-
-            def __init__(self, name):
-                self.name = name
-
-            def lock(self):
-                """Acquire this lock.
-
-                @rtype: C{bool}
-                @return: True if the lock is acquired, false otherwise.
-
-                @raise: Any exception os.symlink() may raise, other than
-                EEXIST.
-                """
-                try:
-                    pid = readlink(self.name)
-                except (OSError, IOError), e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                    self.clean = True
-                else:
-                    if not hasattr(os, 'kill'):
-                        return False
-                    try:
-                        os.kill(int(pid), 0)
-                    except (OSError, IOError), e:
-                        if e.errno != errno.ESRCH:
-                            raise
-                        rmlink(self.name)
-                        self.clean = False
-                    else:
-                        return False
-
-                symlink(str(os.getpid()), self.name)
-                self.locked = True
-                return True
-
-            def unlock(self):
-                """Release this lock.
-
-                This deletes the directory with the given name.
-
-                @raise: Any exception os.readlink() may raise, or
-                ValueError if the lock is not owned by this process.
-                """
-                pid = readlink(self.name)
-                if int(pid) != os.getpid():
-                    raise ValueError("Lock %r not owned by this process" % (self.name,))
-                rmlink(self.name)
-                self.locked = False
-
-try:
-    from twisted.python.log import msg as log
-except ImportError:
-    def log(*args):
-        pass
-
-
-# max number of bytes that can be indexed without forcing an index
-# flush. this limits memory consumption
-MAX_DATA_INDEXED_BETWEEN_FLUSHES = 200 * 1000
-
-MAX_DOCS_TO_RETURN = 1000 * 1000
-
-XAPIAN_LOCK_FILENAME = "db_lock"
-XAPWRAP_LOCK_FILENAME = "xapian_lock"
-
-# Xapian error handling is somewhat weak: all errors trigger either an
-# IOError, a RuntimeError, or a ValueError. The exception's args
-# attribute is a singleton tuple containing an explanation
-# string. Possible errors include 'DatabaseCorruptError: Quartz metafile
-# /tmp/foo/meta is invalid: magic string not found.' and
-# 'DatabaseLockError: Unable to acquire database write lock
-# /tmp/foo/db_lock'. Instead of looking inside exception error strings
-# everywhere, I made a wrapper for xapian database operations that
-# catches exceptions and translates them into the more meaningful
-# exceptions shown below.
-
-class XapianError(StandardError):
-    pass
-class XapianRuntimeError(XapianError):
-    pass
-class XapianLogicError(XapianError):
-    pass
-class XapianDatabaseError(XapianError):
-    pass
-
-class XapianAssertionError(XapianLogicError):
-    pass
-class InvalidOperationError(XapianLogicError):
-    pass
-class InvalidArgumentError(XapianLogicError):
-    pass
-class UnimplementedError(XapianLogicError):
-    pass
-
-class DocNotFoundError(XapianRuntimeError):
-    pass
-class RangeError(XapianRuntimeError):
-    pass
-class InternalError(XapianRuntimeError):
-    pass
-class FeatureUnavalableError(XapianRuntimeError):
-    pass
-class XapianNetworkError(XapianRuntimeError):
-    pass
-
-class NetworkTimeoutError(XapianNetworkError):
-    pass
-
-class DatabaseCorruptionError(XapianDatabaseError):
-    pass
-class DatabaseCreationError(XapianDatabaseError):
-    pass
-class DatabaseOpeningError(XapianDatabaseError):
-    pass
-class DatabaseLockError(XapianDatabaseError):
-    pass
-class DatabaseModifiedError(XapianDatabaseError):
-    pass
-
-# these exceptions are not Xapian errors
-class UnknownDatabaseError(XapianError):
-    pass
-
-class NoIndexValueFound(XapianError):
-    pass
-
-class InconsistantIndex(XapianError):
-    pass
-
-class InconsistantIndexCombination(XapianError):
-    pass
-
-
-def makeTranslatedMethod(methodName):
-    def translatedMethod(self, *args, **kwargs):
-        try:
-            return getattr(self.db, methodName)(*args, **kwargs)
-        except (IOError, RuntimeError, ValueError), e:
-            errorMsg = e.args[0]
-            for subString, exceptionClass in self.exceptionStrMap.iteritems():
-                if subString in errorMsg:
-                    raise exceptionClass(e)
-            else:
-                raise UnknownDatabaseError(e)
-        except:
-            raise
-    return translatedMethod
-
-class ExceptionTranslater:
-    def __init__(self, db):
-        self.db = db
-
-    def openIndex(klass, readOnly, *args, **kwargs):
-        try:
-            if readOnly:
-                assert len(kwargs) == 0
-                # assume all args are db paths
-                db = xapian.Database(args[0])
-                for path in args[1:]:
-                    db.add_database(xapian.Database(path))
-                return klass(db)
-            else:
-                try:
-                    return klass(xapian.WritableDatabase(*args, **kwargs)) # for xapian 1.0+
-                except AttributeError:
-                    return klass(xapian.open(*args, **kwargs)) # for xapian 0.9.x
-        except (IOError, RuntimeError, ValueError), e:
-            errorMsg = e.args[0]
-            for subString, exceptionClass in klass.exceptionStrMap.iteritems():
-                if subString in errorMsg:
-                    raise exceptionClass(e)
-            else:
-                raise UnknownDatabaseError(e)
-        except Exception, e:
-            raise UnknownDatabaseError(e)
-
-    openIndex = classmethod(openIndex)
-
-    # possible exceptions are taken from the list at
-    # http://www.xapian.org/docs/apidoc/html/errortypes_8h.html
-    exceptionStrMap = {
-        # exceptions whose names differ between xapwrap and Xapian
-        'DatabaseCorruptError': DatabaseCorruptionError,
-        'AssertionError': XapianAssertionError,
-        'DatabaseCreateError': DatabaseCreationError,
-
-        # exceptions translated with the same name
-        'DatabaseLockError': DatabaseLockError,
-        'DatabaseOpeningError': DatabaseOpeningError,
-        'DatabaseModifiedError': DatabaseModifiedError,
-        'FeatureUnavalableError': FeatureUnavalableError,
-        'DocNotFoundError': DocNotFoundError,
-        'InvalidOperationError': InvalidOperationError,
-        'InvalidArgumentError': InvalidArgumentError,
-        'UnimplementedError': UnimplementedError,
-        'NetworkError': XapianNetworkError,
-        'NetworkTimeoutError': NetworkTimeoutError,
-        'DatabaseError': XapianDatabaseError,
-        'InternalError': InternalError,
-        'RangeError': RangeError,
-        'RuntimeError': XapianRuntimeError,
-        'LogicError': XapianLogicError
-        }
-
-    get_doccount = makeTranslatedMethod('get_doccount')
-    add_document = makeTranslatedMethod('add_document')
-    replace_document = makeTranslatedMethod('replace_document')
-    delete_document = makeTranslatedMethod('delete_document')
-    flush = makeTranslatedMethod('flush')
-    term_exists = makeTranslatedMethod('term_exists')
-    reopen = makeTranslatedMethod('reopen')
-    begin_transaction = makeTranslatedMethod('begin_transaction')
-    commit_transaction = makeTranslatedMethod('commit_transaction')
-    cancel_transaction = makeTranslatedMethod('cancel_transaction')
-    get_lastdocid = makeTranslatedMethod('get_lastdocid')
-    get_avlength = makeTranslatedMethod('get_avlength')
-    get_termfreq = makeTranslatedMethod('get_termfreq')
-    get_collection_freq = makeTranslatedMethod('get_collection_freq')
-    get_doclength = makeTranslatedMethod('get_doclength')
-    get_document = makeTranslatedMethod('get_document')
-
-    postlist_begin = makeTranslatedMethod('postlist_begin')
-    postlist_end = makeTranslatedMethod('postlist_end')
-    termlist_begin = makeTranslatedMethod('termlist_begin')
-    termlist_end = makeTranslatedMethod('termlist_end')
-    positionlist_begin = makeTranslatedMethod('positionlist_begin')
-    positionlist_end = makeTranslatedMethod('positionlist_end')
-    allterms_begin = makeTranslatedMethod('allterms_begin')
-    allterms_end = makeTranslatedMethod('allterms_end')
-
-
-def makeProtectedDBMethod(method, setupDB = True):
-    def protectedMethod(self, *args, **kwargs):
-        if setupDB:
-            self.setupDB()
-        try:
-            return method(self, *args, **kwargs)
-##         # test that this works and doesn't recurse infinitely
-##         except DatabaseModifiedError:
-##             self.reopen()
-##             return protectedMethod(self, *args, **kwargs)
-        except XapianError, e:
-            #log("error encountered while performing xapian index operation %s: %s"
-            #    % (method.__name__, e))
-            self.close()
-            raise
-    return protectedMethod
-
-
-# there are lots of places below where we write code like:
-# enq = mset = None
-# try:
-#     enq = self.enquire(foo)
-#     mset = enq.get_mset(0, 10)
-#     return mset[0][flimflam]
-# except:
-#     del enq, mset
-#     raise
-
-# the purpose of this code is to ensure that no references to enquire
-# objects or msets will outlive the function call. msets and enquire
-# objsects hold a reference to the xapian db, and thus prevent it from
-# being properly gc'd. if we fail to delete enq and mset on exception,
-# then they can be kept around for arbitrarily long periods of time as
-# part of the exception state
-
-
-# be extremely careful about keeping a db object in local scope;
-# once its there, an unhandled exception could create a traceback
-# containing a frame object that holds a copy of the locals dict,
-# including the db object. if that frame/traceback object is kept
-# around forever (which parts of twisted/quotient seem to do,
-# especially deferreds), then the db object will never be deleted
-# and the indexer lock will never go away.
-
-# in order to prevent that from happening, we maintain two invariants:
-
-# 1. the db is only accessed as an instance attribute and is never
-# copied into a local variable. i.e., we always say self.db and
-# never ever say db = self.db. this keeps the db object from ever
-# getting captured by a frame/traceback.
-
-# 2. the db is only accessed from within an exception handler that
-# calls self.close() in the event of *any* failure. this ensures
-# that the instance loses all references to the db on failure, so,
-# even if the instance object is captured by a frame object (or
-# something else), the db will already have been freed.
-
-
-class ReadOnlyIndex:
-    """
-    I represent a Xapian index that is read only by wrapping the
-    xapian.Database class. Because I provide read only access, I can be
-    used to combine several Xapian indices into one index with
-    performance only slightly lower than when using only one index.
-
-    @cvar DEFAULT_QUERY_COMBINER_OP: the operation used by the query parser to combine query terms
-
-    @cvar STEMMING_LANGUAGE: the language used by the query parser for
-    stemming. this is of little use since Xapwrap does not yet support
-    stemming when indexing.
-
-    @ivar names: a sequence of file names representing paths to Xapian
-    indices
-
-    Please use the configure method to modify C{prefixMap} and C{indexValueMap}
-
-    @ivar prefixMap: a map of prefixes used by named fields in the index
-    and the name they should be referred to by the query parser
-
-    @ivar indexValueMap: a map from sort field names to value integer
-
-    @ivar amountIndexedSinceLastFlush: the number of bytes indexed since
-    the last flush
-
-    The following instance attributes should never be modified or
-    accessed directly:
-
-    @ivar db: the xapian index object
-    @ivar qp: the xapian query parser object
-    @ivar _searchSessions: a map from query description string to
-    (enquire, lastIndexSortedBy)
-    """
-
-    DEFAULT_QUERY_COMBINER_OP = xapian.Query.OP_AND
-    STEMMING_LANGUAGE = 'none'
-
-    def __init__(self, *names):
-        if len(names) < 1:
-            raise ValueError("No index directory supplied to Index constructor")
-        self.names = names
-        self.db = None
-        self.qp = None
-        self._searchSessions = {}
-        self.prefixMap = {}
-        self.indexValueMap = {}
-        self.amountIndexedSinceLastFlush = 0
-
-    def setupDB(self):
-        # we hide the db so that methods always access it only through
-        # this method since db objects can be silently reaped when not
-        # in use. db objects consume 5 file descriptors.
-
-        if self.db is None:
-            self._setupDB()
-
-            #self.qp = xapian.QueryParser()
-            # this is vital: these options specify no language for
-            # stemming (""), disable stemming (False), and specify an
-            # empty stop word object (None). we need this because by
-            # default, xapian's query parser does english stemming
-            #s = xapian.Stem(self.STEMMING_LANGUAGE)
-            #self.qp.set_stemmer(s)
-
-            # we want query terms to be ANDed together by default
-            #self.qp.set_default_op(self.DEFAULT_QUERY_COMBINER_OP)
-            self._configure()
-
-            log("Index %s contains %s documents" %
-                (self.names, self.get_doccount()))
-
-    def _setupDB(self):
-        self.db = ExceptionTranslater.openIndex(True, *self.names)
-
-    def close(self):
-        log("closing xapian index %s" % self.names)
-        for query in self._searchSessions.keys():
-            del self._searchSessions[query]
-        self.qp = None
-        self.db = None
-
-    def _configure(self):
-        if 'uid' not in self.indexValueMap:
-            # this a gross hack...
-            self.indexValueMap['uid'] = 0
-            self.indexValueMap['uidREV'] = 1
-        if self.qp is not None:
-            for k, v in self.prefixMap.iteritems():
-                # check for unicode encoding?
-                if v:
-                    V = v.upper()
-                else:
-                    V = k.upper()
-                self.qp.add_prefix(k, V)
-
-    def configure(self, prefixMap = None, indexValueMap = None):
-        if prefixMap is not None:
-            self.prefixMap = prefixMap
-        if indexValueMap is not None:
-            self.indexValueMap = indexValueMap
-        self._configure()
-
-    def get_doccount(self):
-        return self.db.get_doccount()
-    get_doccount = makeProtectedDBMethod(get_doccount)
-
-    def enquire(self, query):
-        searchSession = None
-        try:
-            searchSession = xapian.Enquire(self.db.db)
-            searchSession.set_query(query)
-            return searchSession
-        except:
-            del query, searchSession
-            raise
-    enquire = makeProtectedDBMethod(enquire)
-
-    def flush(self):
-        if self.db is not None:
-            self.db.flush()
-            self.amountIndexedSinceLastFlush = 0
-    flush = makeProtectedDBMethod(flush)
-
-    def search(self, query,
-               sortKey = None,
-               startingIndex = 0,
-               batchSize = MAX_DOCS_TO_RETURN,
-               sortIndex = None, sortAscending = True,
-               sortByRelevence = False,
-               valuesWanted = None,
-               collapseKey = None):
-        """
-        Search an index.
-
-        @ivar valuesWanted: a list of Values that will be returned as part
-        of the result dictionary.
-        """
-
-        # TODO - allow a simple way to get Keywords out
-        self.setupDB()
-        if isinstance(query, (str, unicode)):
-            query = ParsedQuery(query)
-        elif not(isinstance(query, Query)):
-            raise ValueError("query %s must be either a string or a "
-                             "subclass of xapwrap.Query" % query)
-
-        q = query.prepare(self.qp)
-        # uggg. this mess is due to the fact that xapain Query objects
-        # don't hash in a sane way.
-        try:
-            qString = q.get_description() # deprecated since xapian 1.0, removal in 1.1
-        except AttributeError:
-            qString = str(q)
-
-        # the only thing we use sortKey for is to set sort index
-        if sortKey is not None:
-            sortIndex = self.indexValueMap[sortKey]
-        if collapseKey is not None:
-            collapseKey = self.indexValueMap[collapseKey]
-
-        # once you call set_sorting on an Enquire instance, there is no
-        # way to resort it by relevence, so we have to open a new
-        # session instead.
-
-        # ignore sortAscending since there's no easy way to implement
-        # ascending relevancy sorts and it's tough to imagine a case
-        # where you'd want to see the worst results. in any event, the
-        # user can always sort by relevancy and go to the last page of
-        # results.
-
-        enq = mset = None
-        if qString not in self._searchSessions:
-            self._searchSessions[qString] = (self.enquire(q), None)
-        try:
-            enq, lastIndexSortedBy = self._searchSessions[qString]
-
-            # if we don't set sortIndex, the results will be returned
-            # sorted by relevance, assuming that we have never called
-            # set_sorting on this session
-            if sortByRelevence and lastIndexSortedBy is not None:
-                sortIndex = sortKey = None
-                if lastIndexSortedBy is not None:
-                    del self._searchSessions[qString]
-                    self._searchSessions[qString] = (self.enquire(q), None)
-                    enq, lastIndexSortedBy = self._searchSessions[qString]
-            if sortByRelevence is not None and sortIndex is not None:
-                enq.set_sort_by_relevance_then_value(sortIndex, not sortAscending)
-            elif sortIndex is not None:
-                # It seems that we have the opposite definition of sort ascending
-                # than Xapian so we invert the ascending flag!
-                enq.set_sort_by_value(sortIndex, not sortAscending)
-
-            if collapseKey is not None:
-                enq.set_collapse_key(collapseKey)
-
-            self._searchSessions[qString] = (enq, sortIndex)
-
-            mset = enq.get_mset(startingIndex, batchSize)
-            results = []
-            for m in mset:
-                thisResult = {}
-                thisResult['uid'] = m[xapian.MSET_DID]
-                thisResult['score'] = m[xapian.MSET_PERCENT]
-                if valuesWanted:
-                    xapDoc = m[4]
-                    valRes = {}
-                    for valName in valuesWanted:
-                        valueIndex = self.indexValueMap.get(valName, None)
-                        if valueIndex is None:
-                            raise NoIndexValueFound(valName, self.indexValueMap)
-                        valRes[valName] = xapDoc.get_value(valueIndex)
-                    thisResult['values'] = valRes
-                results.append(thisResult)
-            return enq, mset, results
-        except:
-            del enq, mset
-            raise
-    search = makeProtectedDBMethod(search)
-
-    def count(self, query):
-        enq = mset = None
-        try:
-            enq = self.enquire(query)
-            # get_matches_estimated does not return accurate results if
-            # given a small ending number like 0 or 1
-            mset = enq.get_mset(0, MAX_DOCS_TO_RETURN)
-            sizeEstimate = mset.get_matches_estimated()
-            return sizeEstimate, self.get_doccount()
-        except:
-            del enq, mset
-            raise
-    count = makeProtectedDBMethod(count)
-
-    def checkIndex(self, maxID):
-        """Compute a list of all UIDs less than or equal to maxID that
-        are not in the db.
-        """
-        # I had originally suspected that the performance hit of
-        # returning a huge list in the case of empty indexes would be
-        # substantial, but testing with a 120,000 msg index indicates
-        # that performance is fine and that the space overhead is quite
-        # reasonable. If that were not the case, this could be optimized
-        # by calculating the maximum document ID in the index and only
-        # scanning up to the minimum of maxID and the max ID in the
-        # index, assuming that were using the same document IDs in the
-        # index as in atop.
-
-        missingUIDs = []
-        for uid in xrange(maxID + 1):
-            term = makePairForWrite('UID', str(uid))
-            if not self.db.term_exists(term):
-                missingUIDs.append(uid)
-        return missingUIDs
-    checkIndex = makeProtectedDBMethod(checkIndex)
-
-    def get_documents(self, uid):
-        """ return a list of remapped UIDs corresponding to the actual UID given
-        """
-        docTerm = makePairForWrite('UID', str(uid))
-        candidates = self.search(RawQuery(docTerm))
-        return [int(c['uid']) for c in candidates]
-
-    def get_document(self, uid):
-        # we cannot simply use db.get_document since doc ids get
-        # remapped when combining multiple databases
-        candidates = self.get_documents(uid)
-        if len(candidates) == 0:
-            raise DocNotFoundError(uid)
-        elif len(candidates) == 1:
-            return self._get_document(candidates[0])
-        else:
-            raise InconsistantIndex(
-                "Something has gone horribly wrong. I tried "
-                "retrieving document id %s but found %i documents "
-                "with that document ID term" % (uid, len(candidates)))
-
-    def _get_document(self, uid):
-        assert isinstance(uid, int)
-        return self.db.get_document(uid)
-    _get_document = makeProtectedDBMethod(_get_document)
-
-    def term_exists(self, term):
-        assert isinstance(term, str)
-        return self.db.term_exists(term)
-    term_exists = makeProtectedDBMethod(term_exists)
-
-    def get_lastdocid(self):
-        return self.db.get_lastdocid()
-    get_lastdocid = makeProtectedDBMethod(get_lastdocid)
-
-# XXX FIXME: we should consider deleting all searchSessions whenever we
-# add a document, or we should reopen the db
-
-
-class Index(ReadOnlyIndex):
-
-    def __init__(self, name, create = False, analyzer = None):
-        # XXX FIXME: we should really try opening the db here, so that
-        # any errors are caught immediately rather than waiting for the
-        # first time we try to do something...
-        ReadOnlyIndex.__init__(self, name)
-        self.name = name
-        if create:
-            self.flags = xapian.DB_CREATE_OR_OPEN
-        else:
-            self.flags = xapian.DB_OPEN
-        self.analyzer = analyzer or StandardAnalyzer()
-        self.lockFile = FilesystemLock(
-            os.path.join(self.name, XAPWRAP_LOCK_FILENAME))
-
-    def _setupDB(self):
-        """ really get a xapian database object """
-
-        # xapian expects directories! self.name should refer to a
-        # directory. if it doesn't exist, we'll make one.
-        if not os.path.exists(self.name):
-            os.mkdir(self.name)
-
-        # try to acquire a lock file
-        if not self.lockFile.lock():
-            owningPid = os.readlink(self.lockFile.name)
-            errorMsg = ("cannot acquire lock file for xapian index %s"
-                        "because it is owned by process %s" %
-                        (self.name, owningPid))
-            log(errorMsg)
-            raise DatabaseLockError(errorMsg)
-        xapLockFilePath = os.path.join(self.name, XAPIAN_LOCK_FILENAME)
-        if os.path.exists(xapLockFilePath):
-            log("Stale database lock found in %s. Deleting it now." % xapLockFilePath)
-            os.remove(xapLockFilePath)
-
-        # actually try to open a xapian DB
-        try:
-            try:
-                self.db = ExceptionTranslater.openIndex(False, self.name, self.flags)
-            except DatabaseCorruptionError, e:
-                # the index is trashed, so there's no harm in blowing it
-                # away and starting from scratch
-                log("Xapian index at %s is corrupted and will be destroyed"
-                    % self.name)
-                if self.lockFile.locked:
-                    self.lockFile.unlock()
-                for idxFname in glob.glob(os.path.join(self.name, '*')):
-                    os.remove(idxFname)
-                self.db = ExceptionTranslater.openIndex(False, self.name, self.flags)
-        finally:
-            if self.db is None and self.lockFile.locked:
-                self.lockFile.unlock()
-
-    def __del__(self):
-        self.close()
-
-    def close(self):
-        # this is important! the only way to get xapian to release the
-        # db lock is to call the db object's destructor. that won't
-        # happen until nobody is holding a reference to the db
-        # object. unfortunately, the query parser holds a reference to
-        # it, so the query parser must also go away. do not hold
-        # references to these objects anywhere but here.
-
-        # enquire objects and mset objects hold a reference to the db,
-        # so if any of them are left alive, the db will not be reclaimed
-
-        if self.db is not None:
-            ReadOnlyIndex.close(self)
-            # the islink test is needed in case the index directory has
-            # been deleted before we close was called.
-            if self.lockFile.locked and os.path.islink(self.lockFile.name):
-                self.lockFile.unlock()
-            # there is no point in checking if the lock file is still
-            # around right here: it will only be deleted when xapian's
-            # destructor runs, but python defers running destructors
-            # until after exception handling is complete. since this
-            # code will often get called from an exception handler, we
-            # have to assume that the lock file's removal will be
-            # delayed at least until after this method exits
-
-    def get_document(self, uid):
-        return self._get_document(uid)
-
-    # methods that modify db state
-
-    def index(self, doc):
-        self.setupDB()
-        if hasattr(doc, 'uid') and doc.uid:
-            uid = int(doc.uid)
-            doc.sortFields.append(SortKey('uid', uid))
-            doc.keywords.append(Keyword('uid', str(uid)))
-            xapDoc = doc.toXapianDocument(self.indexValueMap, self.prefixMap)
-            self.replace_document(uid, xapDoc)
-        else:
-            # We need to know the uid of the doc we're going to add
-            # before we add it so we can setup appropriate uid sorting
-            # values. But, another thread could potentially insert a
-            # document at that uid after we determine the last uid, but
-            # before we manage the insertion. Yay race conditions! So we
-            # try to add the document and then check that it ended up at
-            # the right uid. If it did not, we update it with the
-            # correct uid sort values.
-            uid = self.get_lastdocid() + 1
-            doc.sortFields.append(SortKey('uid', uid))
-            doc.keywords.append(Keyword('uid', str(uid)))
-            xapDoc = doc.toXapianDocument(self.indexValueMap, self.prefixMap)
-            newUID = self.add_document(xapDoc)
-            if newUID != uid:
-                doc.sortFields.append(SortKey('uid', newUID))
-                doc.keywords.append(Keyword('uid', str(newUID)))
-                xapDoc = doc.toXapianDocument(self.indexValueMap, self.prefixMap)
-                self.replace_document(newUID, xapDoc)
-
-            # a simpler alternative would be to add an empty document
-            # and then replace it. the problem with that strategy is
-            # that it kills performance since xapian performs an
-            # implicit flush when you replace a document that was added
-            # but not yet committed to disk.
-
-        self.amountIndexedSinceLastFlush += len(doc)
-        if self.amountIndexedSinceLastFlush > MAX_DATA_INDEXED_BETWEEN_FLUSHES:
-            self.flush()
-        return uid
-
-    def add_document(self, doc):
-        return self.db.add_document(doc)
-    add_document = makeProtectedDBMethod(add_document)
-
-    def replace_document(self, uid, doc):
-        return self.db.replace_document(uid, doc)
-    replace_document = makeProtectedDBMethod(replace_document)
-
-    def delete_document(self, docID):
-        return self.db.delete_document(docID)
-    delete_document = makeProtectedDBMethod(delete_document)
-
-class Query:
-    pass
-
-class ParsedQuery(Query):
-    def __init__(self, queryString):
-        if isinstance(queryString, unicode):
-            queryString = queryString.encode(UNICODE_ENCODING, UNICODE_ERROR_POLICY)
-        # as of xapian 0.9.5 the query parser makes trouble with utf-8. but it
-        # also doesnt work with iso-8859-15, so we just live with ascii-only search
-        # for now... - a utf8 fix seems to be planned for the near future!
-        self.queryString = queryString
-
-    def prepare(self, queryParser):
-        return queryParser.parse_query(self.queryString)
-
-class RawQuery(Query):
-    def __init__(self, queryString):
-        if isinstance(queryString, unicode):
-            queryString = queryString.encode('utf-8')
-
-        assert isinstance(queryString, str)
-        self.queryString = queryString
-
-    def prepare(self, queryParser):
-        return xapian.Query(self.queryString)
-
-class QObjQuery(Query):
-    def __init__(self, query):
-        assert isinstance(query, xapian.Query)
-        self.query = query
-
-    def prepare(self, queryParser):
-        return self.query
-
-class SmartIndex(Index):
-    documentFactory = Document
-
-    def __init__(self, *args, **kwargs):
-        Index.__init__(self, *args, **kwargs)
-        self.fetchState()
-
-    def saveState(self):
-        self.setupDB()
-        state = {'indexValueMap': self.indexValueMap,
-                 'prefixMap': self.prefixMap}
-        d = self.documentFactory(uid = 1, data = state)
-        self.index(d, checkID = False)
-        self.flush()
-
-    def fetchState(self):
-        self.setupDB()
-        if self.get_doccount() == 0:
-            # Don't rely on the try:except: for this case
-            self.saveState()
-        try:
-            doc = self.get_document(1)
-        except DocNotFoundError:
-            newState = {'indexValueMap': {}, 'prefixMap': {}}
-            self.saveState()
-        else:
-            dataStr = doc.get_data()
-            newState = cPickle.loads(dataStr)
-        self.indexValueMap.update(newState['indexValueMap'])
-        self.prefixMap.update(newState['prefixMap'])
-
-    def index(self, doc, checkID = True):
-        if hasattr(doc, 'uid') and doc.uid == 1 and checkID:
-            raise InvalidArgumentError(
-                "document UIDs must be greater than one when using SmartIndex")
-
-        docSortKeys = set([sk.name for sk in doc.sortFields if sk.name is not None])
-        indexSortKeys = set(self.indexValueMap.keys())
-        if not docSortKeys.issubset(indexSortKeys):
-            nextValueIndex = 1 + max(self.indexValueMap.itervalues())
-            # we sort the sortKeys in order to improve the odds that two
-            # indices that are indexed with the same documents in the
-            # same order will always end up with the same
-            # indexValueMaps, even if different versions of python are
-            # used with different hash functions
-            sortKeys = list(docSortKeys)
-            sortKeys.sort()
-            for sortKey in sortKeys:
-                if sortKey not in self.indexValueMap:
-                    assert nextValueIndex % 2 == 0
-                    self.indexValueMap[sortKey] = nextValueIndex
-                    self.indexValueMap[sortKey + 'REV'] = nextValueIndex + 1
-                    nextValueIndex += 2
-            self.saveState()
-
-        docKeywords = set([tf.name for tf in doc.textFields if tf.prefix] +
-                               [kw.name for kw in doc.keywords])
-        indexKeyWords = set(self.prefixMap.keys())
-        if not docKeywords.issubset(indexKeyWords):
-            for k in docKeywords - indexKeyWords:
-                self.prefixMap[k] = k.upper()
-            self.saveState()
-
-        return Index.index(self, doc)
-
-
-class SmartReadOnlyIndex(ReadOnlyIndex):
-
-    def __init__(self, *args, **kwargs):
-        ReadOnlyIndex.__init__(self, *args, **kwargs)
-        self.fetchState()
-
-    def fetchState(self):
-        stateDocIDs = self.get_documents(1)
-        stateDocs = map(self._get_document, stateDocIDs)
-        states = [cPickle.loads(s.get_data()) for s in stateDocs]
-
-        # should we issue a warning when the number of states that we
-        # retrieve is less than the number of indices we opened? the
-        # only problem is that some indices may be empty, but there's no
-        # easy way to check how many documents are in a subindex without
-        # opening it explicitly using xapian.Database and that seems
-        # rather expensive for this code path.
-
-        # merge all the states into a master state
-        master = {'prefixMap': self.prefixMap,
-                  'indexValueMap': self.indexValueMap}
-        # note that if there are conflicts, there is no guarantee on who
-        # will win, but it doesn't matter since we'll die on conflicts
-        # later anyway
-        for s in states:
-            for substate in ('prefixMap', 'indexValueMap'):
-                sub = s.get(substate, {})
-                mSub = master[substate]
-                for k, v in sub.iteritems():
-                    mSub[k] = v
-
-        # ensure that states are compatible (check for conflicts)
-        conflicts = []
-        for s in states:
-            for substate in ('prefixMap', 'indexValueMap'):
-                sub = s.get(substate, {})
-                mSub = master[substate]
-                for k, v in sub.iteritems():
-                    if k in mSub and mSub[k] != v:
-                        # we defer error reporting so that the user sees
-                        # as much info on the error as possible
-                        conflicts.append((substate, k, v, mSub[k]))
-
-        # the only way states can be incompatible is if two states have
-        # different values for the same keys in the same substate
-
-        if conflicts:
-            raise InconsistantIndexCombination(
-                "The SmartReadOnlyIndex opened on %s cannot recconcile "
-                "the following conflicts in the subindices' states:\n%s"
-                % (self.names,
-                   '\n'.join(["%s[%r] is %r in one index but %r in another"
-                              % c for c in conflicts])))
-
-        self.prefixMap = master['prefixMap']
-        self.indexValueMap = master['indexValueMap']
-
-    def search(self, query, sortKey = None,
-               startingIndex = 0,
-               batchSize = MAX_DOCS_TO_RETURN,
-               sortIndex = None, sortAscending = True,
-               sortByRelevence = False):
-        # if the appropriate index value string is not in
-        # self.indexValueMap, fetchState() before calling
-        # ReadOnlyIndex.search. if it still isn't there, let
-        # ReadOnlyIndex.search take care of throwing an error
-        if sortKey is not None and sortKey not in self.indexValueMap:
-            self.fetchState()
-        return ReadOnlyIndex.search(self, query, sortKey,
-                                    startingIndex, batchSize,
-                                    sortIndex, sortAscending,
-                                    sortByRelevence)
-