make whoosh storage type and params configurable, updated docs
authorThomas Waldmann <tw AT waldmann-edv DOT de>
Sat, 08 Sep 2012 23:57:40 +0200
changeset 17804a054063fbb0
parent 1779 082581e8688c
child 1781 90eb2ea82c32
make whoosh storage type and params configurable, updated docs

Currently it only supports:

index_storage = 'FileStorage', (index_dir, ), {}

But it is now easy to add other types of whoosh storages, see
MoinMoin/storage/middleware/indexing.py (search for WHOOSH_FILESTORAGE).
MoinMoin/_tests/wikiconfig.py
MoinMoin/app.py
MoinMoin/storage/middleware/_tests/test_serialization.py
MoinMoin/storage/middleware/indexing.py
docs/admin/index.rst
wikiconfig.py
     1.1 --- a/MoinMoin/_tests/wikiconfig.py	Sat Sep 08 21:21:57 2012 +0200
     1.2 +++ b/MoinMoin/_tests/wikiconfig.py	Sat Sep 08 23:57:40 2012 +0200
     1.3 @@ -19,7 +19,7 @@
     1.4      _here = abspath(dirname(__file__))
     1.5      _root = abspath(join(_here, '..', '..'))
     1.6      data_dir = join(_here, 'wiki', 'data') # needed for plugins package TODO
     1.7 -    index_dir = join(_here, 'wiki', 'index')
     1.8 +    index_storage = 'FileStorage', (join(_here, 'wiki', 'index'), ), {}
     1.9      content_acl = None
    1.10      item_root = 'FrontPage'
    1.11      interwikiname = u'MoinTest'
     2.1 --- a/MoinMoin/app.py	Sat Sep 08 21:21:57 2012 +0200
     2.2 +++ b/MoinMoin/app.py	Sat Sep 08 23:57:40 2012 +0200
     2.3 @@ -168,7 +168,7 @@
     2.4      if app.cfg.create_storage:
     2.5          app.router.create()
     2.6      app.router.open()
     2.7 -    app.storage = indexing.IndexingMiddleware(app.cfg.index_dir, app.router,
     2.8 +    app.storage = indexing.IndexingMiddleware(app.cfg.index_storage, app.router,
     2.9                                                wiki_name=app.cfg.interwikiname,
    2.10                                                acl_rights_contents=app.cfg.acl_rights_contents)
    2.11      if app.cfg.create_index:
     3.1 --- a/MoinMoin/storage/middleware/_tests/test_serialization.py	Sat Sep 08 21:21:57 2012 +0200
     3.2 +++ b/MoinMoin/storage/middleware/_tests/test_serialization.py	Sat Sep 08 23:57:40 2012 +0200
     3.3 @@ -10,7 +10,7 @@
     3.4  
     3.5  from StringIO import StringIO
     3.6  
     3.7 -from ..indexing import IndexingMiddleware
     3.8 +from ..indexing import IndexingMiddleware, WHOOSH_FILESTORAGE
     3.9  from ..routing import Backend as RoutingBackend
    3.10  from ..serialization import serialize, deserialize
    3.11  
    3.12 @@ -58,7 +58,7 @@
    3.13      request.addfinalizer(backend.destroy)
    3.14      request.addfinalizer(backend.close)
    3.15  
    3.16 -    mw = IndexingMiddleware(index_dir=str(tmpdir/'foo'),
    3.17 +    mw = IndexingMiddleware(index_storage=(WHOOSH_FILESTORAGE, (str(tmpdir/'foo'), ), {}),
    3.18                              backend=backend)
    3.19      mw.create()
    3.20      mw.open()
     4.1 --- a/MoinMoin/storage/middleware/indexing.py	Sat Sep 08 21:21:57 2012 +0200
     4.2 +++ b/MoinMoin/storage/middleware/indexing.py	Sat Sep 08 23:57:40 2012 +0200
     4.3 @@ -92,6 +92,7 @@
     4.4  from MoinMoin.storage.error import NoSuchItemError, ItemAlreadyExistsError
     4.5  
     4.6  
     4.7 +WHOOSH_FILESTORAGE = 'FileStorage'
     4.8  INDEXES = [LATEST_REVS, ALL_REVS, ]
     4.9  
    4.10  
    4.11 @@ -201,12 +202,11 @@
    4.12  
    4.13  
    4.14  class IndexingMiddleware(object):
    4.15 -    def __init__(self, index_dir, backend, wiki_name=None, acl_rights_contents=[], **kw):
    4.16 +    def __init__(self, index_storage, backend, wiki_name=None, acl_rights_contents=[], **kw):
    4.17          """
    4.18          Store params, create schemas.
    4.19          """
    4.20 -        self.index_dir = index_dir
    4.21 -        self.index_dir_tmp = index_dir + '.temp'
    4.22 +        self.index_storage = index_storage
    4.23          self.backend = backend
    4.24          self.wikiname = wiki_name
    4.25          self.ix = {}  # open indexes
    4.26 @@ -323,23 +323,36 @@
    4.27          # or latest revs index):
    4.28          self.common_fields = set(latest_revs_fields.keys()) & set(all_revs_fields.keys())
    4.29  
    4.30 +    def get_storage_params(self, tmp=False):
    4.31 +        kind, params, kw = self.index_storage
    4.32 +        params, kw = list(params), dict(kw)  # better make a (mutable) copy
    4.33 +        if kind == WHOOSH_FILESTORAGE:
    4.34 +            # index_storage = 'FileStorage', (index_dir, ), {}
    4.35 +            if tmp:
    4.36 +                params[0] += '.temp'
    4.37 +            from whoosh.filedb.filestore import FileStorage
    4.38 +            cls = FileStorage
    4.39 +        else:
    4.40 +            raise ValueError("index_storage = {0!r} is not supported!".format(kind))
    4.41 +        return kind, cls, params, kw
    4.42 +
    4.43      def get_storage(self, tmp=False, create=False):
    4.44          """
    4.45          Get the whoosh storage (whoosh supports different kinds of storage,
    4.46          e.g. to filesystem or to GAE).
    4.47          Currently we only support the FileStorage.
    4.48          """
    4.49 -        from whoosh.filedb.filestore import FileStorage
    4.50 -        index_dir = self.index_dir_tmp if tmp else self.index_dir
    4.51 -        if create:
    4.52 -            try:
    4.53 -                os.mkdir(index_dir)
    4.54 -            except:
    4.55 -                # ignore exception, we'll get another exception below
    4.56 -                # in case there are problems with the index_dir
    4.57 -                pass
    4.58 -        storage = FileStorage(index_dir)
    4.59 -        return storage
    4.60 +        kind, cls, params, kw = self.get_storage_params(tmp)
    4.61 +        if kind == WHOOSH_FILESTORAGE:
    4.62 +            if create:
    4.63 +                index_dir = params[0]
    4.64 +                try:
    4.65 +                    os.mkdir(index_dir)
    4.66 +                except:
    4.67 +                    # ignore exception, we'll get another exception below
    4.68 +                    # in case there are problems with the index_dir
    4.69 +                    pass
    4.70 +        return cls(*params, **kw)
    4.71  
    4.72      def open(self):
    4.73          """
    4.74 @@ -370,17 +383,23 @@
    4.75          Destroy all indexes.
    4.76          """
    4.77          # XXX this is whoosh backend specific and currently only works for FileStorage.
    4.78 -        index_dir = self.index_dir_tmp if tmp else self.index_dir
    4.79 -        if os.path.exists(index_dir):
    4.80 -            shutil.rmtree(index_dir)
    4.81 +        kind, cls, params, kw = self.get_storage_params(tmp)
    4.82 +        if kind == WHOOSH_FILESTORAGE:
    4.83 +            index_dir = params[0]
    4.84 +            if os.path.exists(index_dir):
    4.85 +                shutil.rmtree(index_dir)
    4.86  
    4.87      def move_index(self):
    4.88          """
    4.89 -        Move freshly built indexes from index_dir_tmp to index_dir.
    4.90 +        Move freshly built indexes from tmp storage to normal storage
    4.91          """
    4.92          # XXX this is whoosh backend specific and currently only works for FileStorage.
    4.93 -        self.destroy()
    4.94 -        os.rename(self.index_dir_tmp, self.index_dir)
    4.95 +        kind, cls, params, kw = self.get_storage_params(False)
    4.96 +        if kind == WHOOSH_FILESTORAGE:
    4.97 +            _, _, params_tmp, _ = self.get_storage_params(True)
    4.98 +            self.destroy()
    4.99 +            index_dir, index_dir_tmp = params[0], params_tmp[0]
   4.100 +            os.rename(index_dir_tmp, index_dir)
   4.101  
   4.102      def index_revision(self, meta, content, async=True):
   4.103          """
     5.1 --- a/docs/admin/index.rst	Sat Sep 08 21:21:57 2012 +0200
     5.2 +++ b/docs/admin/index.rst	Sat Sep 08 23:57:40 2012 +0200
     5.3 @@ -19,15 +19,23 @@
     5.4  
     5.5  Configuration
     5.6  =============
     5.7 -Your wiki config needs ``index_dir`` to point to a writable directory. A fast,
     5.8 -local filesystem is preferred.
     5.9 -Use something like::
    5.10 +Your need to have a ``index_storage`` entry in your wiki config.
    5.11  
    5.12 -    index_dir = "/path/to/moin-2.0/wiki/index"
    5.13 +We use whoosh for indexing and as whoosh supports multiple storage backends,
    5.14 +this entry is made to potentially support any storage supported by whoosh.
    5.15  
    5.16 -**Note:**
    5.17 -* The path MUST be absolute.
    5.18 -* Moin will use `index_dir`.temp location as well, if you build an index at
    5.19 +In general, this entry has the form of::
    5.20 +
    5.21 +    index_storage = kind, (p1, p2, ...), {kw1=..., kw2=..., ...}
    5.22 +
    5.23 +Currently, we only support the 'FileStorage' kind of index storage, which only
    5.24 +has one parameter - the index directory::
    5.25 +
    5.26 +    index_storage = 'FileStorage', ("/path/to/moin-2.0/wiki/index", ), {}
    5.27 +
    5.28 +**Notes for FileStorage:**
    5.29 +* The path MUST be absolute, writable and should be on a fast, local filesystem.
    5.30 +* Moin will use `index.temp` directory as well, if you build an index at
    5.31  the `temporary location`.
    5.32  
    5.33  
    5.34 @@ -143,12 +151,14 @@
    5.35  ``Sales``::
    5.36  
    5.37        interwikiname = u"Sales"
    5.38 -      index_dir = "/path/to/wiki/index"
    5.39 +      index_storage = 'FileStorage', ("/path/to/moin-2.0/wiki/index", ), {}
    5.40 +
    5.41  
    5.42  ``Engineering``::
    5.43  
    5.44        interwikiname = u"Engineering"
    5.45 -      index_dir = "/path/to/wiki/index"
    5.46 +      index_storage = 'FileStorage', ("/path/to/moin-2.0/wiki/index", ), {}
    5.47 +
    5.48  
    5.49  Now do the initial index building::
    5.50  
     6.1 --- a/wikiconfig.py	Sat Sep 08 21:21:57 2012 +0200
     6.2 +++ b/wikiconfig.py	Sat Sep 08 23:57:40 2012 +0200
     6.3 @@ -26,7 +26,7 @@
     6.4      # If that's not true, feel free to adjust the pathes.
     6.5      instance_dir = os.path.join(wikiconfig_dir, 'wiki')
     6.6      data_dir = os.path.join(instance_dir, 'data') # Note: this used to have a trailing / in the past
     6.7 -    index_dir = os.path.join(instance_dir, "index")
     6.8 +    index_storage = 'FileStorage', (os.path.join(instance_dir, "index"), ), {}
     6.9  
    6.10      # This provides a simple default setup for your backend configuration.
    6.11      # 'stores:fs:...' indicates that you want to use the filesystem backend.