changeset 370:459fda5f2079

Add wikiname field support for runtime indexing and index script, now wiki admin can create wikis farm using one app.cfg.index_dir
author Michael Mayorov <marchael@kb.csu.ru>
date Wed, 13 Jul 2011 02:13:50 +0000
parents 5c6ef2031235
children fd28cf720d2c
files MoinMoin/app.py MoinMoin/script/maint/index.py MoinMoin/search/indexing.py MoinMoin/storage/backends/indexing.py
diffstat 4 files changed, 48 insertions(+), 29 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/app.py	Tue Jul 12 21:48:30 2011 +0000
+++ b/MoinMoin/app.py	Wed Jul 13 02:13:50 2011 +0000
@@ -175,18 +175,15 @@
     # mountpoint, unprotected backend, protection to apply as a dict
     ns_mapping = app.cfg.namespace_mapping
     index_uri = app.cfg.router_index_uri
-    index_dir = app.cfg.index_dir
     # Just initialize with unprotected backends.
     unprotected_mapping = [(ns, backend) for ns, backend, acls in ns_mapping]
     unprotected_storage = router.RouterBackend(unprotected_mapping, index_uri=index_uri,
-                                               index_dir=index_dir
+                                               cfg=app.cfg
                                               )
     # Protect each backend with the acls provided for it in the mapping at position 2
     amw = acl.AclWrapperBackend
     protected_mapping = [(ns, amw(app.cfg, backend, **acls)) for ns, backend, acls in ns_mapping]
-    storage = router.RouterBackend(protected_mapping, index_uri=index_uri,
-                                   index_dir=index_dir
-                                  )
+    storage = router.RouterBackend(protected_mapping, index_uri=index_uri, cfg=app.cfg)
     return unprotected_storage, storage
 
 def deinit_backends(app):
@@ -200,9 +197,9 @@
     xmlfile = app.cfg.load_xml
     if xmlfile:
         app.cfg.load_xml = None
-        tmp_backend = router.RouterBackend([('/', memory.MemoryBackend())],
-                                           index_uri='sqlite://',
-                                           index_dir=app.cfg.index_dir
+        index_uri = 'sqlite://'
+        tmp_backend = router.RouterBackend([('/', memory.MemoryBackend())], index_uri=index_uri,
+                                           cfg=app.cfg
                                           )
         unserialize(tmp_backend, xmlfile)
         # TODO optimize this, maybe unserialize could count items it processed
--- a/MoinMoin/script/maint/index.py	Tue Jul 12 21:48:30 2011 +0000
+++ b/MoinMoin/script/maint/index.py	Wed Jul 13 02:13:50 2011 +0000
@@ -99,7 +99,9 @@
                     for item, rev_no in latest_documents:
                         revision = item.get_revision(rev_no)
                         converted_rev = backend_to_index(revision, rev_no, latest_rev_field_names)
-                        found = latest_rev_searcher.document(name_exact=item.name)
+                        found = latest_rev_searcher.document(name_exact=item.name,
+                                                             wikiname=app.cfg.wiki_name
+                                                            )
                         if not found:
                             latest_rev_writer.add_document(**converted_rev)
                         # Checking what last revision is the latest
@@ -113,7 +115,8 @@
                     for item, rev_nos in delete_documents:
                         for rev_no in rev_nos:
                             doc_number = all_rev_searcher.document_number(rev_no=rev_no,
-                                                                          exact_name=item.name
+                                                                          exact_name=item.name,
+                                                                          wikiname=app.cfg.wiki_name
                                                                          )
                             if doc_number:
                                 all_rev_writer.delete_document(doc_number)
@@ -177,7 +180,7 @@
             """
             Return list of found documents for given name using index searcher
             """
-            revs_found = searcher.documents(name_exact=name)
+            revs_found = searcher.documents(name_exact=name, wikiname=app.cfg.wiki_name)
             return [rev["rev_no"] for rev in revs_found]
 
         def backend_to_index(backend_rev, rev_no, schema_fields):
@@ -190,6 +193,7 @@
             metadata[MTIME] = datetime.datetime.fromtimestamp(metadata[MTIME])
             metadata["name_exact"] = backend_rev[NAME]
             metadata["rev_no"] = rev_no
+            metadata["wikiname"] = app.cfg.wiki_name 
             metadata["content"] = convert_data(backend_rev, rev_no)
             return metadata
 
--- a/MoinMoin/search/indexing.py	Tue Jul 12 21:48:30 2011 +0000
+++ b/MoinMoin/search/indexing.py	Wed Jul 13 02:13:50 2011 +0000
@@ -19,10 +19,10 @@
 
 
 class WhooshIndex(object):
-    """ 
-    Creating\opeining whoosh indexes using "latest_revisions_schema" and
+    """
+    Creating\opening whoosh indexes using "latest_revisions_schema" and
     "all_revisions_schema"
-    """"
+    """
 
     latest_revisions_schema = Schema(wikiname=ID(stored=True),
                                      name=TEXT(stored=True, analyzer=item_name_analyzer()),
--- a/MoinMoin/storage/backends/indexing.py	Tue Jul 12 21:48:30 2011 +0000
+++ b/MoinMoin/storage/backends/indexing.py	Wed Jul 13 02:13:50 2011 +0000
@@ -40,9 +40,9 @@
     """
     def __init__(self, *args, **kw):
         index_uri = kw.pop('index_uri', None)
-        index_dir = kw.pop('index_dir', None)
+        cfg = kw.pop('cfg', None)
         super(IndexingBackendMixin, self).__init__(*args, **kw)
-        self._index = ItemIndex(index_uri, index_dir)
+        self._index = ItemIndex(index_uri, cfg)
 
     def close(self):
         self._index.close()
@@ -220,7 +220,7 @@
     """
     Index for Items/Revisions
     """
-    def __init__(self, index_uri, index_dir):
+    def __init__(self, index_uri, cfg):
         metadata = MetaData()
         metadata.bind = create_engine(index_uri, echo=False)
 
@@ -259,7 +259,8 @@
         self.item_kvstore = KVStore(item_kvmeta)
         self.rev_kvstore = KVStore(rev_kvmeta)
 
-        self.index_object = WhooshIndex(index_dir=index_dir)
+        self.wikiname = cfg.wiki_name
+        self.index_object = WhooshIndex(index_dir=cfg.index_dir)
 
     def close(self):
         engine = self.metadata.bind
@@ -293,7 +294,7 @@
 
     def get_item_id_whoosh(self, uuid):
         with self.index_object.latest_revisions_index.searcher() as searcher:
-            result = searcher.document(uuid=uuid)
+            result = searcher.document(uuid=uuid, wikiname=self.wikiname)
         if result:
             return result
 
@@ -315,7 +316,9 @@
 
     def update_item_whoosh(self, metas):
         with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            doc_number = latest_revs_searcher.document_number(uuid=metas[UUID])
+            doc_number = latest_revs_searcher.document_number(uuid=metas[UUID],
+                                                              wikiname=self.wikiname
+                                                             )
         with AsyncWriter(self.index_object.latest_revisions_index) as async_writer:
             if doc_number:
                 async_writer.delete_document(doc_number)
@@ -351,7 +354,8 @@
     def remove_item_whoosh(self, metas):
         with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
             doc_number = latest_revs_searcher.document_number(uuid=metas[UUID],
-                                                              name_exact=metas[NAME]
+                                                              name_exact=metas[NAME],
+                                                              wikiname=self.wikiname
                                                              )
         if doc_number is not None:
             with AsyncWriter(self.index_object.latest_revisions_index) as async_writer:
@@ -386,9 +390,14 @@
 
     def add_rev_whoosh(self, uuid, revno, metas):
         with self.index_object.all_revisions_index.searcher() as all_revs_searcher:
-            all_found_document = all_revs_searcher.document(uuid=metas[UUID], rev_no=revno)
+            all_found_document = all_revs_searcher.document(uuid=metas[UUID],
+                                                            rev_no=revno,
+                                                            wikiname=self.wikiname
+                                                           )
         with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            latest_found_document = latest_revs_searcher.document(uuid=metas[UUID])
+            latest_found_document = latest_revs_searcher.document(uuid=metas[UUID],
+                                                                  wikiname=self.wikiname
+                                                                 )
         logging.debug("To add: uuid %s revno %s" % (metas[UUID], revno))
         if not all_found_document:
             field_names = self.index_object.all_revisions_index.schema.names()
@@ -429,9 +438,15 @@
 
     def remove_rev_whoosh(self, uuid, revno):
         with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            latest_doc_number = latest_revs_searcher.document_number(uuid=uuid, rev_no=revno)
+            latest_doc_number = latest_revs_searcher.document_number(uuid=uuid,
+                                                                     rev_no=revno,
+                                                                     wikiname=self.wikiname
+                                                                    )
         with self.index_object.all_revisions_index.searcher() as all_revs_searcher:
-            doc_number = all_revs_searcher.document_number(uuid=uuid, rev_no=revno)
+            doc_number = all_revs_searcher.document_number(uuid=uuid,
+                                                           rev_no=revno,
+                                                           wikiname=self.wikiname
+                                                          )
         if doc_number is not None:
             with AsyncWriter(self.index_object.all_revisions_index) as async_writer:
                 logging.debug("REMOVE FROM ALL: %d", doc_number)
@@ -492,9 +507,11 @@
             mountpoint += '/'
         with self.index_object.all_revisions_index.searcher() as all_revs_searcher:
             if item_name:
-                docs = all_revs_searcher.documents(name_exact=item_name)
+                docs = all_revs_searcher.documents(name_exact=item_name,
+                                                   wikiname=self.wikiname
+                                                  )
             else:
-                docs = all_revs_searcher.all_stored_fields()
+                docs = all_revs_searcher.documents(wikiname=self.wikiname)
             for doc in sorted(docs, reverse=reverse)[start:end]:
                 yield (doc[MTIME], mountpoint + doc[NAME], doc["rev_no"])
 
@@ -511,7 +528,7 @@
 
     def all_tags_whoosh(self):
         with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            docs = latest_revs_searcher.all_stored_fields()
+            docs = latest_revs_searcher.documents(wikiname=self.wikiname)
             tags_names = {}
             for doc in docs:
                 tags = doc.get(TAGS, [])
@@ -529,7 +546,7 @@
 
     def tagged_items_whoosh(self, tag):
         with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            docs = latest_revs_searcher.documents(tags=tag)
+            docs = latest_revs_searcher.documents(tags=tag, wikiname=self.wikiname)
             return [doc[NAME] for doc in docs]
 
     def backend_to_index(self, backend_rev, rev_no, schema_fields):
@@ -542,6 +559,7 @@
         metadata[MTIME] = datetime.datetime.fromtimestamp(metadata[MTIME])
         metadata["name_exact"] = backend_rev[NAME]
         metadata["rev_no"] = rev_no
+        metadata["wikiname"] = self.wikiname
         metadata["content"] = self.convert_data(backend_rev, rev_no)
         return metadata