changeset 1785:41eb0ef524e2 gae

make moin work for GAE (google app engine), details see below Most of this stuff is based on work done by Guido van Rossum, thanks! added a GAE store implementation for moin wikiconfig: adjusted so it works with GAE by default (in this branch): * use moin's GAE store * use whoosh's GAE backend logging: currently needs a hack (configured=True), otherwise it just hangs strangely. added a app.yaml definition file for GAE added a appengine_main.py script for usage with GAE Notes: * there needs to be a "support" directory with all of moin's dependencies (except jinja2, sphinx, py.test) on the same level as "MoinMoin" == in the top level directory. * we currently need the 2.5 development version of whoosh. * I did not apply the patch for filesys.py as we currently do not import this module at all, so there is no problem with it on GAE either.
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Fri, 21 Sep 2012 23:05:41 +0200
parents 4e236dcf39cd
children c4892659dbee
files MoinMoin/log.py MoinMoin/storage/middleware/indexing.py MoinMoin/storage/stores/gae.py app.yaml appengine_main.py wikiconfig.py
diffstat 6 files changed, 134 insertions(+), 3 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/log.py	Fri Sep 21 07:43:57 2012 +0200
+++ b/MoinMoin/log.py	Fri Sep 21 23:05:41 2012 +0200
@@ -95,7 +95,10 @@
 import logging.config
 import logging.handlers  # needed for handlers defined there being configurable in logging.conf file
 
-configured = False
+# TODO: configured should be False of course, but this crashes GAE!
+# maybe this is related how we (ab)use and patch the logger as "logging".
+configured = True
+
 fallback_config = False
 
 import warnings
--- a/MoinMoin/storage/middleware/indexing.py	Fri Sep 21 07:43:57 2012 +0200
+++ b/MoinMoin/storage/middleware/indexing.py	Fri Sep 21 23:05:41 2012 +0200
@@ -93,6 +93,7 @@
 
 
 WHOOSH_FILESTORAGE = 'FileStorage'
+WHOOSH_GAE = 'DatastoreStorage'
 INDEXES = [LATEST_REVS, ALL_REVS, ]
 
 
@@ -332,6 +333,9 @@
                 params[0] += '.temp'
             from whoosh.filedb.filestore import FileStorage
             cls = FileStorage
+        elif kind == WHOOSH_GAE:
+            from whoosh.filedb.gae import DatastoreStorage
+            cls = DatastoreStorage
         else:
             raise ValueError("index_storage = {0!r} is not supported!".format(kind))
         return kind, cls, params, kw
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/MoinMoin/storage/stores/gae.py	Fri Sep 21 23:05:41 2012 +0200
@@ -0,0 +1,87 @@
+# Copyright: 2011 MoinMoin:GuidoVanRossum
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - Google App Engine store
+
+Store into Google App Engine datastore (using NDB), one entity per k/v pair.
+"""
+
+from __future__ import absolute_import, division
+
+import cStringIO as StringIO
+import logging
+
+from google.appengine.ext import ndb
+
+from . import MutableStoreBase, BytesMutableStoreBase, FileMutableStoreBase
+
+
+class _MoinDirectory(ndb.Model):
+    """Used as a parent key."""
+
+
+class _MoinValue(ndb.Model):
+    """Used to store a value.
+
+    The parent is a _MoinDirectory key (but no _MoinDirectory
+    object exists).
+    """
+
+    value = ndb.BlobProperty()
+
+
+class _Store(MutableStoreBase):
+    """Keys and uris are required to be valid key names.
+
+    (This is not an onerous requirement.)
+    """
+
+    @classmethod
+    def from_uri(cls, uri):
+        return cls(uri)
+
+    def __init__(self, path):
+        logging.info('%s(%r)', self.__class__.__name__, path)
+        self._root_key = ndb.Key(_MoinDirectory, path)
+        self._query = _MoinValue.query(ancestor=self._root_key)
+
+    def create(self):
+        """Nothing to do."""
+
+    def destroy(self):
+        self._query.map(self._destroy_key, keys_only=True)
+
+    def _destroy_key(self, key):
+        return key.delete()
+
+    def __iter__(self):
+        return self._query.iter(keys_only=True)
+
+    def _getitem(self, key):
+        ent = _MoinValue.get_by_id(key, parent=self._root_key)
+        return ent and ent.value or 'null'
+
+    def _setitem(self, key, value):
+        _MoinValue(value=value, id=key, parent=self._root_key).put()
+
+    def __delitem__(self, key):
+        ndb.Key(_MoinValue, key, parent=self._root_key).delete()
+
+
+class BytesStore(_Store, BytesMutableStoreBase):
+
+    def __getitem__(self, key):
+        return self._getitem(key)
+    
+    def __setitem__(self, key, value):
+        self._setitem(key, value)
+
+
+class FileStore(_Store, FileMutableStoreBase):
+
+    def __getitem__(self, key):
+        return StringIO.StringIO(self._getitem(key))
+
+    def __setitem__(self, key, stream):
+        return self._setitem(key, stream.read())
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/app.yaml	Fri Sep 21 23:05:41 2012 +0200
@@ -0,0 +1,14 @@
+application: moin2-test
+version: dev
+runtime: python27
+api_version: 1
+threadsafe: true
+
+handlers:
+- url: /.*
+  script: appengine_main.application
+
+libraries:
+- name: jinja2
+  version: latest
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/appengine_main.py	Fri Sep 21 23:05:41 2012 +0200
@@ -0,0 +1,23 @@
+"""Main entry point for Google App Engine."""
+
+# Python imports.
+import os
+import sys
+
+# Configuration constants.
+wiki_config = 'wikiconfig.py'
+
+# Tweak sys.path.
+support_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'support'))
+if support_path not in sys.path:
+    sys.path.insert(0, support_path)
+
+# Now we can import MoinMoin.
+from MoinMoin.app import create_app
+
+# Hack: If there are no DatastoreFile instances assume we must create the index.
+from whoosh.filedb.gae import DatastoreFile
+create_index = DatastoreFile.all().get() is None
+
+# Create the WSGI application object.
+application = create_app(os.path.abspath(wiki_config), create_index)
--- a/wikiconfig.py	Fri Sep 21 07:43:57 2012 +0200
+++ b/wikiconfig.py	Fri Sep 21 23:05:41 2012 +0200
@@ -26,12 +26,12 @@
     # If that's not true, feel free to adjust the pathes.
     instance_dir = os.path.join(wikiconfig_dir, 'wiki')
     data_dir = os.path.join(instance_dir, 'data') # Note: this used to have a trailing / in the past
-    index_storage = 'FileStorage', (os.path.join(instance_dir, "index"), ), {}
+    index_storage = 'DatastoreStorage', (), {}
 
     # This provides a simple default setup for your backend configuration.
     # 'stores:fs:...' indicates that you want to use the filesystem backend.
     namespace_mapping, acl_mapping = create_simple_mapping(
-                            uri='stores:fs:{0}/%(nsname)s/%(kind)s'.format(data_dir),
+                            uri='stores:gae:/%(nsname)s/%(kind)s',
                             # XXX we use rather relaxed ACLs for the development wiki:
                             content_acl=dict(before=u'',
                                              default=u'All:read,write,create,destroy,admin',